lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

trees and forests update

Browse Source
This commit is contained in:
mhjensen
2019-10-24 06:16:45 +02:00
parent 3c9acac906
commit 28e4865ba6
38 changed files with 2563 additions and 3235 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/pub/DecisionTrees/html/._DecisionTrees-bs000.html
+55 -57
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -178,7 +176,7 @@ MathJax.Hub.Config({
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
<br>
<p>
<center><h4>Dec 17, 2018</h4></center> <!-- date -->
<center><h4>Oct 24, 2019</h4></center> <!-- date -->
<br>
<p>
@@ -202,7 +200,7 @@ MathJax.Hub.Config({
<li><a href="._DecisionTrees-bs008.html">9</a></li>
<li><a href="._DecisionTrees-bs009.html">10</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs001.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
@@ -220,7 +218,7 @@ MathJax.Hub.Config({
<center style="font-size:80%">
<!-- copyright --> &copy; 1999-2018, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
<!-- copyright --> &copy; 1999-2019, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
</center>
doc/pub/DecisionTrees/html/._DecisionTrees-bs001.html
+53 -55
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -207,7 +205,7 @@ and <b>leaf nodes</b> which are then connected by <b>branches</b>.
<li><a href="._DecisionTrees-bs009.html">10</a></li>
<li><a href="._DecisionTrees-bs010.html">11</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs002.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs002.html
+53 -55
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -192,7 +190,7 @@ Then we are essentially done!
<li><a href="._DecisionTrees-bs010.html">11</a></li>
<li><a href="._DecisionTrees-bs011.html">12</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs003.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs003.html
+53 -55
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -271,7 +269,7 @@ plt<span style="color: #666666">.</span>show()
<li><a href="._DecisionTrees-bs011.html">12</a></li>
<li><a href="._DecisionTrees-bs012.html">13</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs004.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs004.html
+74 -145
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,104 +157,35 @@ MathJax.Hub.Config({
<a name="part0004"></a>
<!-- !split -->
<h2 id="___sec3" class="anchor">Maxwell-Boltzmann velocity distribution </h2>
<h2 id="___sec3" class="anchor">Building a tree, regression </h2>
<p>
There are mainly two steps
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Program to test the Metropolis algorithm with one particle at given temp in</span>
<span style="color: #408080; font-style: italic"># one dimension</span>
<span style="color: #408080; font-style: italic">#!/usr/bin/env python</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.mlab</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">mlab</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">random</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">math</span> <span style="color: #008000; font-weight: bold">import</span> sqrt, exp, log
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> PolynomialFeatures
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LinearRegression
<span style="color: #408080; font-style: italic"># initialize the rng with a seed</span>
random<span style="color: #666666">.</span>seed()
<span style="color: #408080; font-style: italic"># Hard coding of input parameters</span>
MCcycles <span style="color: #666666">=</span> <span style="color: #666666">100000</span>
Temperature <span style="color: #666666">=</span> <span style="color: #666666">2.0</span>
beta <span style="color: #666666">=</span> <span style="color: #666666">1./</span>Temperature
InitialVelocity <span style="color: #666666">=</span> <span style="color: #666666">-2.0</span>
CurrentVelocity <span style="color: #666666">=</span> InitialVelocity
Energy <span style="color: #666666">=</span> <span style="color: #666666">0.5*</span>InitialVelocity<span style="color: #666666">*</span>InitialVelocity
VelocityRange <span style="color: #666666">=</span> <span style="color: #666666">10*</span>sqrt(Temperature)
VelocityStep <span style="color: #666666">=</span> <span style="color: #666666">2*</span>VelocityRange<span style="color: #666666">/10.</span>
AverageEnergy <span style="color: #666666">=</span> Energy
AverageEnergy2 <span style="color: #666666">=</span> Energy<span style="color: #666666">*</span>Energy
VelocityValues <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(MCcycles)
<span style="color: #408080; font-style: italic"># The Monte Carlo sampling with Metropolis starts here</span>
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span> (<span style="color: #666666">1</span>, MCcycles, <span style="color: #666666">1</span>):
TrialVelocity <span style="color: #666666">=</span> CurrentVelocity <span style="color: #666666">+</span> (<span style="color: #666666">2.0*</span>random<span style="color: #666666">.</span>random() <span style="color: #666666">-</span> <span style="color: #666666">1.0</span>)<span style="color: #666666">*</span>VelocityStep
EnergyChange <span style="color: #666666">=</span> <span style="color: #666666">0.5*</span>(TrialVelocity<span style="color: #666666">*</span>TrialVelocity <span style="color: #666666">-</span>CurrentVelocity<span style="color: #666666">*</span>CurrentVelocity);
<span style="color: #008000; font-weight: bold">if</span> random<span style="color: #666666">.</span>random() <span style="color: #666666">&lt;=</span> exp(<span style="color: #666666">-</span>beta<span style="color: #666666">*</span>EnergyChange):
CurrentVelocity <span style="color: #666666">=</span> TrialVelocity
Energy <span style="color: #666666">+=</span> EnergyChange
VelocityValues[i] <span style="color: #666666">=</span> CurrentVelocity
AverageEnergy <span style="color: #666666">+=</span> Energy
AverageEnergy2 <span style="color: #666666">+=</span> Energy<span style="color: #666666">*</span>Energy
<span style="color: #408080; font-style: italic">#Final averages</span>
AverageEnergy <span style="color: #666666">=</span> AverageEnergy<span style="color: #666666">/</span>MCcycles
AverageEnergy2 <span style="color: #666666">=</span> AverageEnergy2<span style="color: #666666">/</span>MCcycles
Variance <span style="color: #666666">=</span> AverageEnergy2 <span style="color: #666666">-</span> AverageEnergy<span style="color: #666666">*</span>AverageEnergy
<span style="color: #008000; font-weight: bold">print</span>(AverageEnergy, Variance)
n, bins, patches <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>hist(VelocityValues, <span style="color: #666666">400</span>, facecolor<span style="color: #666666">=</span><span style="color: #BA2121">&#39;green&#39;</span>)
<ol>
<li> We split the predictor space (the set of possible values \( x_1,x_2,\dots, x_p \)) into \( J \)</li>
</ol>
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&#39;$v$&#39;</span>)
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">&#39;Velocity distribution P(v)&#39;</span>)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">r&#39;Velocity histogram at $k_BT=2$&#39;</span>)
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">-5</span>, <span style="color: #666666">5</span>, <span style="color: #666666">0</span>, <span style="color: #666666">600</span>])
plt<span style="color: #666666">.</span>grid(<span style="color: #008000">True</span>)
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">collections</span> <span style="color: #008000; font-weight: bold">import</span> Counter
distinct and non-non-overlapping regions, \( R_1,R_2,\dots,R_J \).
<span style="color: #408080; font-style: italic">#print (Counter(VelocityValues))</span>
<ol>
<li> For every observation that falls into the region \( R_j \) , we make the same prediction, which is simply the mean of the response values for the training observations in \( R_j \).</li>
</ol>
<span style="color: #008000; font-weight: bold">print</span> (VelocityValues[:<span style="color: #666666">20</span>])
VelocityValues<span style="color: #666666">=</span><span style="color: #008000">list</span>(Counter(VelocityValues)<span style="color: #666666">.</span>keys())
d<span style="color: #666666">=</span><span style="color: #008000">list</span>(Counter(VelocityValues)<span style="color: #666666">.</span>values())
How do we construct the regions \( R_1,\dots,R_J \)?
In theory, the regions could have any shape. However, we
choose to divide the predictor space into high-dimensional rectangles,
or boxes, for simplicity and for ease of interpretation of the
resulting predic- tive model. The goal is to find boxes \( R_1,\dots,R_J \)
that minimize the MSE, given by
$$
\sum_{j=1}^J\sum_{i\in R_j}(y_i-\overline{y}_{R_j})^2,
$$
VelocityValues<span style="color: #666666">=</span>np<span style="color: #666666">.</span>asarray(VelocityValues)[:, np<span style="color: #666666">.</span>newaxis]
d<span style="color: #666666">=</span>np<span style="color: #666666">.</span>asarray(d)
<span style="color: #008000; font-weight: bold">print</span> (VelocityValues<span style="color: #666666">.</span>shape, d<span style="color: #666666">.</span>shape)
where \( \overline{y}_{R_j} \) is the mean response for the training observations
within the $j$th
box.
plt<span style="color: #666666">.</span>scatter(VelocityValues, d)
plt<span style="color: #666666">.</span>show()
<span style="color: #408080; font-style: italic">#2nd Degree Polynomial</span>
poly_feat<span style="color: #666666">=</span>PolynomialFeatures(degree<span style="color: #666666">=20</span>, include_bias<span style="color: #666666">=</span><span style="color: #008000">False</span>)
X_poly<span style="color: #666666">=</span>poly_feat<span style="color: #666666">.</span>fit_transform(VelocityValues)
lin_reg<span style="color: #666666">=</span>LinearRegression()
poly_fit<span style="color: #666666">=</span>lin_reg<span style="color: #666666">.</span>fit(X_poly,d)
y_plot<span style="color: #666666">=</span>poly_fit<span style="color: #666666">.</span>predict(X_poly)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;Polynomial Fit&quot;</span>)
plt<span style="color: #666666">.</span>plot(VelocityValues, y_plot, color<span style="color: #666666">=</span><span style="color: #BA2121">&#39;black&#39;</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;Fit&quot;</span>)
plt<span style="color: #666666">.</span>show()
<span style="color: #408080; font-style: italic">#Decision Trees</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeRegressor
regr_1<span style="color: #666666">=</span>DecisionTreeRegressor(max_depth<span style="color: #666666">=2</span>)
regr_2<span style="color: #666666">=</span>DecisionTreeRegressor(max_depth<span style="color: #666666">=5</span>)
regr_3<span style="color: #666666">=</span>DecisionTreeRegressor(max_depth<span style="color: #666666">=7</span>)
regr_1<span style="color: #666666">.</span>fit(VelocityValues, d)
regr_2<span style="color: #666666">.</span>fit(VelocityValues, d)
regr_3<span style="color: #666666">.</span>fit(VelocityValues, d)
X_test <span style="color: #666666">=</span> np<span style="color: #666666">.</span>arange(<span style="color: #666666">0.0</span>, MCcycles, <span style="color: #666666">0.01</span>)[:, np<span style="color: #666666">.</span>newaxis]
y_1<span style="color: #666666">=</span>regr_1<span style="color: #666666">.</span>predict(X_test)
y_2<span style="color: #666666">=</span>regr_2<span style="color: #666666">.</span>predict(X_test)
y_3<span style="color: #666666">=</span>regr_3<span style="color: #666666">.</span>predict(X_test)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;Decision Tree&quot;</span>)
plt<span style="color: #666666">.</span>plot(X_test, y_1, color<span style="color: #666666">=</span><span style="color: #BA2121">&quot;red&quot;</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;max_depth=2&quot;</span>, linewidth<span style="color: #666666">=2</span>)
plt<span style="color: #666666">.</span>plot(X_test, y_2, color<span style="color: #666666">=</span><span style="color: #BA2121">&quot;green&quot;</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;max_depth=5&quot;</span>, linewidth<span style="color: #666666">=2</span>)
plt<span style="color: #666666">.</span>plot(X_test, y_3, color<span style="color: #666666">=</span><span style="color: #BA2121">&quot;m&quot;</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;max_depth=7&quot;</span>, linewidth<span style="color: #666666">=2</span>)
plt<span style="color: #666666">.</span>show()
</pre></div>
<p>
<p>
<!-- navigation buttons at the bottom of the page -->
@@ -277,7 +206,7 @@ plt<span style="color: #666666">.</span>show()
<li><a href="._DecisionTrees-bs012.html">13</a></li>
<li><a href="._DecisionTrees-bs013.html">14</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs005.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs005.html
+65 -80
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,34 +157,21 @@ MathJax.Hub.Config({
<a name="part0005"></a>
<!-- !split -->
<h2 id="___sec4" class="anchor">Building a tree, regression </h2>
<h2 id="___sec4" class="anchor">A top-down approach, recursive binary splitting </h2>
<p>
There are mainly two steps
Unfortunately, it is computationally infeasible to consider every
possible partition of the feature space into \( J \) boxes.
The common strategy is to take a top-down approach
<ol>
<li> We split the predictor space (the set of possible values \( x_1,x_2,\dots, x_p \)) into \( J \)</li>
</ol>
distinct and non-non-overlapping regions, \( R_1,R_2,\dots,R_J \).
<ol>
<li> For every observation that falls into the region \( R_j \) , we make the same prediction, which is simply the mean of the response values for the training observations in \( R_j \).</li>
</ol>
How do we construct the regions \( R_1,\dots,R_J \)?
In theory, the regions could have any shape. However, we
choose to divide the predictor space into high-dimensional rectangles,
or boxes, for simplicity and for ease of interpretation of the
resulting predic- tive model. The goal is to find boxes \( R_1,\dots,R_J \)
that minimize the MSE, given by
$$
\sum_{j=1}^J\sum_{i\in R_j}(y_i-\overline{y}_{R_j})^2,
$$
where \( \overline{y}_{R_j} \) is the mean response for the training observations
within the $j$th
box.
<p>
The approach is top-down because it begins at the top of the tree (all
observations belong to a single region) and then successively splits
the predictor space; each split is indicated via two new branches
further down on the tree. It is greedy because at each step of the
tree-building process, the best split is made at that particular step,
rather than looking ahead and picking a split that will lead to a
better tree in some future step.
<p>
<p>
@@ -209,7 +194,7 @@ box.
<li><a href="._DecisionTrees-bs013.html">14</a></li>
<li><a href="._DecisionTrees-bs014.html">15</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs006.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs006.html
+92 -66
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,21 +157,49 @@ MathJax.Hub.Config({
<a name="part0006"></a>
<!-- !split -->
<h2 id="___sec5" class="anchor">A top-down approach, recursive binary splitting </h2>
<h2 id="___sec5" class="anchor">Making a tree </h2>
<p>
Unfortunately, it is computationally infeasible to consider every
possible partition of the feature space into \( J \) boxes.
The common strategy is to take a top-down approach
In order to implement the recursive binary splitting we start by selecting
the predictor \( x_j \) and a cutpoint \( s \) that splits the predictor space into two regions \( R_1 \) and \( R_2 \)
$$
\left\{X\vert x_j < s\right\},
$$
and
$$
\left\{X\vert x_j \geq s\right\},
$$
so that we obtain the lowest MSE, that is
$$
\sum_{i:x_i\in R_j}(y_i-\overline{y}_{R_1})^2+\sum_{i:x_i\in R_2}(y_i-\overline{y}_{R_2})^2,
$$
which we want to minimize by considering all predictors \( x_1,x_2,\dots,x_p \).
We consider also all possible values of \( s \) for each predictor. These values could be determined by randomly assigned numbers or by starting at the midpoint and then proceed till we find an optimal value.
<p>
The approach is top-down because it begins at the top of the tree (all
observations belong to a single region) and then successively splits
the predictor space; each split is indicated via two new branches
further down on the tree. It is greedy because at each step of the
tree-building process, the best split is made at that particular step,
rather than looking ahead and picking a split that will lead to a
better tree in some future step.
For any \( j \) and \( s \), we define the pair of
half-planes where \( \overline{y}_{R_1} \) is the mean response for the training
observations in \( R_1(j,s) \), and \( \overline{y}_{R_2} \) is the mean response for the
training observations in \( R_2(j,s) \).
<p>
Finding the values of j and s that
minimize the above equation can be done quite quickly, especially when the number
of features \( p \) is not too large.
<p>
Next, we repeat the process, looking
for the best predictor and best cutpoint in order to split the data
further so as to minimize the MSE within each of the resulting
regions. However, this time, instead of splitting the entire predictor
space, we split one of the two previously identified regions. We now
have three regions. Again, we look to split one of these three regions
further, so as to minimize the MSE. The process continues until a
stopping criterion is reached; for instance, we may continue until no
region contains more than five observations.
<p>
<p>
@@ -197,7 +223,7 @@ better tree in some future step.
<li><a href="._DecisionTrees-bs014.html">15</a></li>
<li><a href="._DecisionTrees-bs015.html">16</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs007.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs007.html
+65 -95
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -157,51 +155,23 @@ MathJax.Hub.Config({
<p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
<a name="part0007"></a>
<!-- !split -->
<!-- !split -->
<h2 id="___sec6" class="anchor">Making a tree </h2>
<h2 id="___sec6" class="anchor">Pruning the tree </h2>
<p>
In order to implement the recursive binary splitting we start by selecting
the predictor \( x_j \) and a cutpoint \( s \) that splits the predictor space into two regions \( R_1 \) and \( R_2 \)
$$
\left\{X\vert x_j < s\right\},
$$
and
$$
\left\{X\vert x_j \geq s\right\},
$$
so that we obtain the lowest MSE, that is
$$
\sum_{i:x_i\in R_j}(y_i-\overline{y}_{R_1})^2+\sum_{i:x_i\in R_2}(y_i-\overline{y}_{R_2})^2,
$$
which we want to minimize by considering all predictors \( x_1,x_2,\dots,x_p \).
We consider also all possible values of \( s \) for each predictor. These values could be determined by randomly assigned numbers or by starting at the midpoint and then proceed till we find an optimal value.
The above procedure is rather straightforward, but leads often to
overfitting and unnecessarily large and complicated trees. The basic
idea is to grow a large tree \( T_0 \) and then prune it back in order to
obtain a subtree. A smaller tree with fewer splits (fewer regions) can
lead to smaller variance and better interpretation at the cost of a
little more bias.
<p>
For any \( j \) and \( s \), we define the pair of
half-planes where \( \overline{y}_{R_1} \) is the mean response for the training
observations in \( R_1(j,s) \), and \( \overline{y}_{R_2} \) is the mean response for the
training observations in \( R_2(j,s) \).
<p>
Finding the values of j and s that
minimize the above equation can be done quite quickly, especially when the number
of features \( p \) is not too large.
<p>
Next, we repeat the process, looking
for the best predictor and best cutpoint in order to split the data
further so as to minimize the MSE within each of the resulting
regions. However, this time, instead of splitting the entire predictor
space, we split one of the two previously identified regions. We now
have three regions. Again, we look to split one of these three regions
further, so as to minimize the MSE. The process continues until a
stopping criterion is reached; for instance, we may continue until no
region contains more than five observations.
The so-called Cost complexity pruning algorithm gives us a
way to do just this. Rather than considering every possible subtree,
we consider a sequence of trees indexed by a nonnegative tuning
parameter \( \alpha \).
<p>
<p>
@@ -226,7 +196,7 @@ region contains more than five observations.
<li><a href="._DecisionTrees-bs015.html">16</a></li>
<li><a href="._DecisionTrees-bs016.html">17</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs008.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs008.html
+77 -67
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -157,23 +155,35 @@ MathJax.Hub.Config({
<p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
<a name="part0008"></a>
<!-- !split -->
<!-- !split -->
<h2 id="___sec7" class="anchor">Pruning the tree </h2>
<h2 id="___sec7" class="anchor">Cost complexity pruning </h2>
For each value of \( \alpha \) there corresponds a subtree \( T \in T_0 \) such that
$$
\sum_{m=1}^{\overline{T}}\sum_{i:x_i\in R_m}(y_i-\overline{y}_{R_m})^2+\alpha\overline{T},
$$
is as small as possible. Here \( \overline{T} \) is
the number of terminal nodes of the tree \( T \) , \( R_m \) is the
rectangle (i.e. the subset of predictor space) corresponding to the \( m \)-th terminal node.
<p>
The above procedure is rather straightforward, but leads often to
overfitting and unnecessarily large and complicated trees. The basic
idea is to grow a large tree \( T_0 \) and then prune it back in order to
obtain a subtree. A smaller tree with fewer splits (fewer regions) can
lead to smaller variance and better interpretation at the cost of a
little more bias.
The tuning parameter \( \alpha \) controls a trade-off between the subtree&#8217;s
com- plexity and its fit to the training data. When \( \alpha = 0 \), then the
subtree \( T \) will simply equal \( T_0 \),
because then the above equation just measures the
training error.
However, as \( \alpha \) increases, there is a price to pay for
having a tree with many terminal nodes. The above equation will
tend to be minimized for a smaller subtree.
<p>
The so-called Cost complexity pruning algorithm gives us a
way to do just this. Rather than considering every possible subtree,
we consider a sequence of trees indexed by a nonnegative tuning
parameter \( \alpha \).
It turns out that as we increase \( \alpha \) from zero
branches get pruned from the tree in a nested and predictable fashion,
so obtaining the whole sequence of subtrees as a function of \( \alpha \) is
easy. We can select a value of \( \alpha \) using a validation set or using
cross-validation. We then return to the full data set and obtain the
subtree corresponding to \( \alpha \).
<p>
<p>
@@ -199,7 +209,7 @@ parameter \( \alpha \).
<li><a href="._DecisionTrees-bs016.html">17</a></li>
<li><a href="._DecisionTrees-bs017.html">18</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs009.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs009.html
+73 -79
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,33 +157,29 @@ MathJax.Hub.Config({
<a name="part0009"></a>
<!-- !split -->
<h2 id="___sec8" class="anchor">Cost complexity pruning </h2>
For each value of \( \alpha \) there corresponds a subtree \( T \in T_0 \) such that
$$
\sum_{m=1}^{\overline{T}}\sum_{i:x_i\in R_m}(y_i-\overline{y}_{R_m})^2+\alpha\overline{T},
$$
is as small as possible. Here \( \overline{T} \) is
the number of terminal nodes of the tree \( T \) , \( R_m \) is the
rectangle (i.e. the subset of predictor space) corresponding to the \( m \)-th terminal node.
<h2 id="___sec8" class="anchor">A schematic procedure </h2>
<p>
The tuning parameter \( \alpha \) controls a trade-off between the subtree&#8217;s
com- plexity and its fit to the training data. When \( \alpha = 0 \), then the
subtree \( T \) will simply equal \( T_0 \),
because then the above equation just measures the
training error.
However, as \( \alpha \) increases, there is a price to pay for
having a tree with many terminal nodes. The above equation will
tend to be minimized for a smaller subtree.
<div class="panel panel-default">
<div class="panel-body">
<p> <!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
<ol>
<li> Use recursive binary splitting to grow a large tree on the training data, stopping only when each terminal node has fewer than some minimum number of observations.</li>
<li> Apply cost complexity pruning to the large tree in order to obtain a sequence of best subtrees, as a function of \( \alpha \).</li>
<li> Use for example \( K \)-fold cross-validation to choose \( \alpha \). Divide the training observations into \( K \) folds. For each \( k=1,2,\dots,K \) we:</li>
<ul>
<li> repeat steps 1 and 2 on all but the \( k \)-th fold of the training data.</li>
<li> Then we valuate the mean squared prediction error on the data in the left-out \( k \)-th fold, as a function of \( \alpha \).</li>
<li> Finally we average the results for each value of \( \alpha \), and pick \( \alpha \) to minimize the average error.</li>
</ul>
<li> Return the subtree from Step 2 that corresponds to the chosen value of \( \alpha \).</li>
</ol>
</div>
</div>
<p>
It turns out that as we increase \( \alpha \) from zero
branches get pruned from the tree in a nested and predictable fashion,
so obtaining the whole sequence of subtrees as a function of \( \alpha \) is
easy. We can select a value of \( \alpha \) using a validation set or using
cross-validation. We then return to the full data set and obtain the
subtree corresponding to \( \alpha \).
<p>
<p>
@@ -212,7 +206,7 @@ subtree corresponding to \( \alpha \).
<li><a href="._DecisionTrees-bs017.html">18</a></li>
<li><a href="._DecisionTrees-bs018.html">19</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs010.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs010.html
+66 -76
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,29 +157,21 @@ MathJax.Hub.Config({
<a name="part0010"></a>
<!-- !split -->
<h2 id="___sec9" class="anchor">A schematic procedure </h2>
<h2 id="___sec9" class="anchor">A classification tree </h2>
<p>
<div class="panel panel-default">
<div class="panel-body">
<p> <!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
<ol>
<li> Use recursive binary splitting to grow a large tree on the training data, stopping only when each terminal node has fewer than some minimum number of observations.</li>
<li> Apply cost complexity pruning to the large tree in order to obtain a sequence of best subtrees, as a function of \( \alpha \).</li>
<li> Use for example \( K \)-fold cross-validation to choose \( \alpha \). Divide the training observations into \( K \) folds. For each \( k=1,2,\dots,K \) we:</li>
<ul>
<li> repeat steps 1 and 2 on all but the \( k \)-th fold of the training data.</li>
<li> Then we valuate the mean squared prediction error on the data in the left-out \( k \)-th fold, as a function of \( \alpha \).</li>
<li> Finally we average the results for each value of \( \alpha \), and pick \( \alpha \) to minimize the average error.</li>
</ul>
<li> Return the subtree from Step 2 that corresponds to the chosen value of \( \alpha \).</li>
</ol>
</div>
</div>
A classification tree is very similar to a regression tree, except
that it is used to predict a qualitative response rather than a
quantitative one. Recall that for a regression tree, the predicted
response for an observation is given by the mean response of the
training observations that belong to the same terminal node. In
contrast, for a classification tree, we predict that each observation
belongs to the most commonly occurring class of training observations
in the region to which it belongs. In interpreting the results of a
classification tree, we are often interested not only in the class
prediction corresponding to a particular terminal node region, but
also in the class proportions among the training observations that
fall into that region.
<p>
<p>
@@ -209,7 +199,7 @@ MathJax.Hub.Config({
<li><a href="._DecisionTrees-bs018.html">19</a></li>
<li><a href="._DecisionTrees-bs019.html">20</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs011.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs011.html
+71 -68
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,21 +157,26 @@ MathJax.Hub.Config({
<a name="part0011"></a>
<!-- !split -->
<h2 id="___sec10" class="anchor">A classification tree </h2>
<h2 id="___sec10" class="anchor">Growing a classification tree </h2>
<p>
A classification tree is very similar to a regression tree, except
that it is used to predict a qualitative response rather than a
quantitative one. Recall that for a regression tree, the predicted
response for an observation is given by the mean response of the
training observations that belong to the same terminal node. In
contrast, for a classification tree, we predict that each observation
belongs to the most commonly occurring class of training observations
in the region to which it belongs. In interpreting the results of a
classification tree, we are often interested not only in the class
prediction corresponding to a particular terminal node region, but
also in the class proportions among the training observations that
fall into that region.
The task of growing a
classification tree is quite similar to the task of growing a
regression tree. Just as in the regression setting, we use recursive
binary splitting to grow a classification tree. However, in the
classification setting, the MSE cannot be used as a criterion for making
the binary splits. A natural alternative to MSE is the <b>classification
error rate</b>. Since we plan to assign an observation in a given region
to the most commonly occurring error rate class of training
observations in that region, the classification error rate is simply
the fraction of the training observations in that region that do not
belong to the most common class.
<p>
When building a classification tree, either the Gini index or the
entropy are typically used to evaluate the quality of a particular
split, since these two approaches are more sensitive to node purity
than is the classification error rate.
<p>
<p>
@@ -201,7 +204,7 @@ fall into that region.
<li><a href="._DecisionTrees-bs019.html">20</a></li>
<li><a href="._DecisionTrees-bs020.html">21</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs012.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs012.html
+87 -71
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,26 +157,44 @@ MathJax.Hub.Config({
<a name="part0012"></a>
<!-- !split -->
<h2 id="___sec11" class="anchor">Growing a classification tree </h2>
<h2 id="___sec11" class="anchor">Classification tree, how to split nodes </h2>
If our targets are the outcome of a classification process that takes for example
\( k=1,2,\dots,K \) values, the only thing we need to think of is to set up the splitting criteria for each node.
<p>
The task of growing a
classification tree is quite similar to the task of growing a
regression tree. Just as in the regression setting, we use recursive
binary splitting to grow a classification tree. However, in the
classification setting, the MSE cannot be used as a criterion for making
the binary splits. A natural alternative to MSE is the <b>classification
error rate</b>. Since we plan to assign an observation in a given region
to the most commonly occurring error rate class of training
observations in that region, the classification error rate is simply
the fraction of the training observations in that region that do not
belong to the most common class.
We define a PDF \( p_{mk} \) that represents the number of observations of a class \( k \) in a region \( R_m \) with \( N_m \) observations. We represent this likelihood function in terms of the proportion \( I(y_i=k) \) of observations of this class in the region \( R_m \) as
$$
p_{mk} = \frac{1}{N_m}\sum_{x_i\in R_m}I(y_i=k).
$$
<p>
When building a classification tree, either the Gini index or the
entropy are typically used to evaluate the quality of a particular
split, since these two approaches are more sensitive to node purity
than is the classification error rate.
We let \( p_{mk} \) represent the majority class of observations in region \( m \). The three most common ways of splitting a node are given by
<ul>
<li> Misclassification error</li>
</ul>
$$
p_{mk} = \frac{1}{N_m}\sum_{x_i\in R_m}I(y_i\ne k) = 1-p_{mk}.
$$
<ul>
<li> Gini index \( g \)</li>
</ul>
$$
g = \sum_{k=1}^K p_{mk}(1-p_{mk}).
$$
<ul>
<li> Information entropy or just entropy \( s \)</li>
</ul>
$$
s = -\sum_{k=1}^K p_{mk}\log{p_{mk}}.
$$
<p>
<p>
@@ -206,7 +222,7 @@ than is the classification error rate.
<li><a href="._DecisionTrees-bs020.html">21</a></li>
<li><a href="._DecisionTrees-bs021.html">22</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs013.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs013.html
+55 -91
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,44 +157,10 @@ MathJax.Hub.Config({
<a name="part0013"></a>
<!-- !split -->
<h2 id="___sec12" class="anchor">Classification tree, how to split nodes </h2>
If our targets are the outcome of a classification process that takes for example
\( k=1,2,\dots,K \) values, the only thing we need to think of is to set up the splitting criteria for each node.
<h2 id="___sec12" class="anchor">Entropy and the ID3 algorithm </h2>
<p>
We define a PDF \( p_{mk} \) that represents the number of observations of a class \( k \) in a region \( R_m \) with \( N_m \) observations. We represent this likelihood function in terms of the proportion \( I(y_i=k) \) of observations of this class in the region \( R_m \) as
$$
p_{mk} = \frac{1}{N_m}\sum_{x_i\in R_m}I(y_i=k).
$$
<p>
We let \( p_{mk} \) represent the majority class of observations in region \( m \). The three most common ways of splitting a node are given by
<ul>
<li> Misclassification error</li>
</ul>
$$
p_{mk} = \frac{1}{N_m}\sum_{x_i\in R_m}I(y_i\ne k) = 1-p_{mk}.
$$
<ul>
<li> Gini index \( g \)</li>
</ul>
$$
g = \sum_{k=1}^K p_{mk}(1-p_{mk}).
$$
<ul>
<li> Information entropy or just entropy \( s \)</li>
</ul>
$$
s = -\sum_{k=1}^K p_{mk}\log{p_{mk}}.
$$
More text to come here.
<p>
<p>
@@ -224,7 +188,7 @@ $$
<li><a href="._DecisionTrees-bs021.html">22</a></li>
<li><a href="._DecisionTrees-bs022.html">23</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs014.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs014.html
+141 -56
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,8 +157,95 @@ MathJax.Hub.Config({
<a name="part0014"></a>
<!-- !split -->
<h2 id="___sec13" class="anchor">Entropy and the ID3 algorithm </h2>
<h2 id="___sec13" class="anchor">Writing your own code for a classification tree </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">entropy</span>(target_col):
<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
<span style="color: #BA2121; font-style: italic"> Calculate the entropy of a dataset.</span>
<span style="color: #BA2121; font-style: italic"> The only parameter of this function is the target_col parameter which specifies the target column</span>
<span style="color: #BA2121; font-style: italic"> &quot;&quot;&quot;</span>
elements,counts <span style="color: #666666">=</span> np<span style="color: #666666">.</span>unique(target_col,return_counts <span style="color: #666666">=</span> <span style="color: #008000">True</span>)
entropy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum([(<span style="color: #666666">-</span>counts[i]<span style="color: #666666">/</span>np<span style="color: #666666">.</span>sum(counts))<span style="color: #666666">*</span>np<span style="color: #666666">.</span>log2(counts[i]<span style="color: #666666">/</span>np<span style="color: #666666">.</span>sum(counts)) <span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(elements))])
<span style="color: #008000; font-weight: bold">return</span> entropy
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">InfoGain</span>(data,split_attribute_name,target_name<span style="color: #666666">=</span><span style="color: #BA2121">&quot;class&quot;</span>):
<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
<span style="color: #BA2121; font-style: italic"> Calculate the information gain of a dataset. This function takes three parameters:</span>
<span style="color: #BA2121; font-style: italic"> 1. data = The dataset for whose feature the IG should be calculated</span>
<span style="color: #BA2121; font-style: italic"> 2. split_attribute_name = the name of the feature for which the information gain should be calculated</span>
<span style="color: #BA2121; font-style: italic"> 3. target_name = the name of the target feature. The default for this example is &quot;class&quot;</span>
<span style="color: #BA2121; font-style: italic"> &quot;&quot;&quot;</span>
<span style="color: #408080; font-style: italic">#Calculate the entropy of the total dataset</span>
total_entropy <span style="color: #666666">=</span> entropy(data[target_name])
<span style="color: #408080; font-style: italic">##Calculate the entropy of the dataset</span>
<span style="color: #408080; font-style: italic">#Calculate the values and the corresponding counts for the split attribute </span>
vals,counts<span style="color: #666666">=</span> np<span style="color: #666666">.</span>unique(data[split_attribute_name],return_counts<span style="color: #666666">=</span><span style="color: #008000">True</span>)
<span style="color: #408080; font-style: italic">#Calculate the weighted entropy</span>
Weighted_Entropy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum([(counts[i]<span style="color: #666666">/</span>np<span style="color: #666666">.</span>sum(counts))<span style="color: #666666">*</span>entropy(data<span style="color: #666666">.</span>where(data[split_attribute_name]<span style="color: #666666">==</span>vals[i])<span style="color: #666666">.</span>dropna()[target_name]) <span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(vals))])
<span style="color: #408080; font-style: italic">#Calculate the information gain</span>
Information_Gain <span style="color: #666666">=</span> total_entropy <span style="color: #666666">-</span> Weighted_Entropy
<span style="color: #008000; font-weight: bold">return</span> Information_Gain
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">ID3</span>(data,originaldata,features,target_attribute_name<span style="color: #666666">=</span><span style="color: #BA2121">&quot;class&quot;</span>,parent_node_class <span style="color: #666666">=</span> <span style="color: #008000">None</span>):
<span style="color: #408080; font-style: italic">#Define the stopping criteria --&gt; If one of this is satisfied, we want to return a leaf node#</span>
<span style="color: #408080; font-style: italic">#If all target_values have the same value, return this value</span>
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">len</span>(np<span style="color: #666666">.</span>unique(data[target_attribute_name])) <span style="color: #666666">&lt;=</span> <span style="color: #666666">1</span>:
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>unique(data[target_attribute_name])[<span style="color: #666666">0</span>]
<span style="color: #408080; font-style: italic">#If the dataset is empty, return the mode target feature value in the original dataset</span>
<span style="color: #008000; font-weight: bold">elif</span> <span style="color: #008000">len</span>(data)<span style="color: #666666">==0</span>:
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>unique(originaldata[target_attribute_name])[np<span style="color: #666666">.</span>argmax(np<span style="color: #666666">.</span>unique(originaldata[target_attribute_name],return_counts<span style="color: #666666">=</span><span style="color: #008000">True</span>)[<span style="color: #666666">1</span>])]
<span style="color: #408080; font-style: italic">#If the feature space is empty, return the mode target feature value of the direct parent node --&gt; Note that</span>
<span style="color: #408080; font-style: italic">#the direct parent node is that node which has called the current run of the ID3 algorithm and hence</span>
<span style="color: #408080; font-style: italic">#the mode target feature value is stored in the parent_node_class variable.</span>
<span style="color: #008000; font-weight: bold">elif</span> <span style="color: #008000">len</span>(features) <span style="color: #666666">==0</span>:
<span style="color: #008000; font-weight: bold">return</span> parent_node_class
<span style="color: #408080; font-style: italic">#If none of the above holds true, grow the tree!</span>
<span style="color: #008000; font-weight: bold">else</span>:
<span style="color: #408080; font-style: italic">#Set the default value for this node --&gt; The mode target feature value of the current node</span>
parent_node_class <span style="color: #666666">=</span> np<span style="color: #666666">.</span>unique(data[target_attribute_name])[np<span style="color: #666666">.</span>argmax(np<span style="color: #666666">.</span>unique(data[target_attribute_name],return_counts<span style="color: #666666">=</span><span style="color: #008000">True</span>)[<span style="color: #666666">1</span>])]
<span style="color: #408080; font-style: italic">#Select the feature which best splits the dataset</span>
item_values <span style="color: #666666">=</span> [InfoGain(data,feature,target_attribute_name) <span style="color: #008000; font-weight: bold">for</span> feature <span style="color: #AA22FF; font-weight: bold">in</span> features] <span style="color: #408080; font-style: italic">#Return the information gain values for the features in the dataset</span>
best_feature_index <span style="color: #666666">=</span> np<span style="color: #666666">.</span>argmax(item_values)
best_feature <span style="color: #666666">=</span> features[best_feature_index]
<span style="color: #408080; font-style: italic">#Create the tree structure. The root gets the name of the feature (best_feature) with the maximum information</span>
<span style="color: #408080; font-style: italic">#gain in the first run</span>
tree <span style="color: #666666">=</span> {best_feature:{}}
<span style="color: #408080; font-style: italic">#Remove the feature with the best inforamtion gain from the feature space</span>
features <span style="color: #666666">=</span> [i <span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> features <span style="color: #008000; font-weight: bold">if</span> i <span style="color: #666666">!=</span> best_feature]
<span style="color: #408080; font-style: italic">#Grow a branch under the root node for each possible value of the root node feature</span>
<span style="color: #008000; font-weight: bold">for</span> value <span style="color: #AA22FF; font-weight: bold">in</span> np<span style="color: #666666">.</span>unique(data[best_feature]):
value <span style="color: #666666">=</span> value
<span style="color: #408080; font-style: italic">#Split the dataset along the value of the feature with the largest information gain and therwith create sub_datasets</span>
sub_data <span style="color: #666666">=</span> data<span style="color: #666666">.</span>where(data[best_feature] <span style="color: #666666">==</span> value)<span style="color: #666666">.</span>dropna()
<span style="color: #408080; font-style: italic">#Call the ID3 algorithm for each of those sub_datasets with the new parameters --&gt; Here the recursion comes in!</span>
subtree <span style="color: #666666">=</span> ID3(sub_data,dataset,features,target_attribute_name,parent_node_class)
<span style="color: #408080; font-style: italic">#Add the sub tree, grown from the sub_dataset to the tree under the root node</span>
tree[best_feature][value] <span style="color: #666666">=</span> subtree
<span style="color: #008000; font-weight: bold">return</span>(tree)
</pre></div>
<p>
<p>
<!-- navigation buttons at the bottom of the page -->
@@ -187,7 +272,7 @@ MathJax.Hub.Config({
<li><a href="._DecisionTrees-bs022.html">23</a></li>
<li><a href="._DecisionTrees-bs023.html">24</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs015.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs015.html
+115 -137
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,94 +157,74 @@ MathJax.Hub.Config({
<a name="part0015"></a>
<!-- !split -->
<h2 id="___sec14" class="anchor">Writing your own code for a classification tree </h2>
<h2 id="___sec14" class="anchor">Back to moons again </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">entropy</span>(target_col):
<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
<span style="color: #BA2121; font-style: italic"> Calculate the entropy of a dataset.</span>
<span style="color: #BA2121; font-style: italic"> The only parameter of this function is the target_col parameter which specifies the target column</span>
<span style="color: #BA2121; font-style: italic"> &quot;&quot;&quot;</span>
elements,counts <span style="color: #666666">=</span> np<span style="color: #666666">.</span>unique(target_col,return_counts <span style="color: #666666">=</span> <span style="color: #008000">True</span>)
entropy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum([(<span style="color: #666666">-</span>counts[i]<span style="color: #666666">/</span>np<span style="color: #666666">.</span>sum(counts))<span style="color: #666666">*</span>np<span style="color: #666666">.</span>log2(counts[i]<span style="color: #666666">/</span>np<span style="color: #666666">.</span>sum(counts)) <span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(elements))])
<span style="color: #008000; font-weight: bold">return</span> entropy
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">__future__</span> <span style="color: #008000; font-weight: bold">import</span> division, print_function, unicode_literals
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">InfoGain</span>(data,split_attribute_name,target_name<span style="color: #666666">=</span><span style="color: #BA2121">&quot;class&quot;</span>):
<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
<span style="color: #BA2121; font-style: italic"> Calculate the information gain of a dataset. This function takes three parameters:</span>
<span style="color: #BA2121; font-style: italic"> 1. data = The dataset for whose feature the IG should be calculated</span>
<span style="color: #BA2121; font-style: italic"> 2. split_attribute_name = the name of the feature for which the information gain should be calculated</span>
<span style="color: #BA2121; font-style: italic"> 3. target_name = the name of the target feature. The default for this example is &quot;class&quot;</span>
<span style="color: #BA2121; font-style: italic"> &quot;&quot;&quot;</span>
<span style="color: #408080; font-style: italic">#Calculate the entropy of the total dataset</span>
total_entropy <span style="color: #666666">=</span> entropy(data[target_name])
<span style="color: #408080; font-style: italic">##Calculate the entropy of the dataset</span>
<span style="color: #408080; font-style: italic">#Calculate the values and the corresponding counts for the split attribute </span>
vals,counts<span style="color: #666666">=</span> np<span style="color: #666666">.</span>unique(data[split_attribute_name],return_counts<span style="color: #666666">=</span><span style="color: #008000">True</span>)
<span style="color: #408080; font-style: italic">#Calculate the weighted entropy</span>
Weighted_Entropy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum([(counts[i]<span style="color: #666666">/</span>np<span style="color: #666666">.</span>sum(counts))<span style="color: #666666">*</span>entropy(data<span style="color: #666666">.</span>where(data[split_attribute_name]<span style="color: #666666">==</span>vals[i])<span style="color: #666666">.</span>dropna()[target_name]) <span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(vals))])
<span style="color: #408080; font-style: italic">#Calculate the information gain</span>
Information_Gain <span style="color: #666666">=</span> total_entropy <span style="color: #666666">-</span> Weighted_Entropy
<span style="color: #008000; font-weight: bold">return</span> Information_Gain
<span style="color: #408080; font-style: italic"># Common imports</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">ID3</span>(data,originaldata,features,target_attribute_name<span style="color: #666666">=</span><span style="color: #BA2121">&quot;class&quot;</span>,parent_node_class <span style="color: #666666">=</span> <span style="color: #008000">None</span>):
<span style="color: #408080; font-style: italic">#Define the stopping criteria --&gt; If one of this is satisfied, we want to return a leaf node#</span>
<span style="color: #408080; font-style: italic">#If all target_values have the same value, return this value</span>
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">len</span>(np<span style="color: #666666">.</span>unique(data[target_attribute_name])) <span style="color: #666666">&lt;=</span> <span style="color: #666666">1</span>:
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>unique(data[target_attribute_name])[<span style="color: #666666">0</span>]
<span style="color: #408080; font-style: italic">#If the dataset is empty, return the mode target feature value in the original dataset</span>
<span style="color: #008000; font-weight: bold">elif</span> <span style="color: #008000">len</span>(data)<span style="color: #666666">==0</span>:
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>unique(originaldata[target_attribute_name])[np<span style="color: #666666">.</span>argmax(np<span style="color: #666666">.</span>unique(originaldata[target_attribute_name],return_counts<span style="color: #666666">=</span><span style="color: #008000">True</span>)[<span style="color: #666666">1</span>])]
<span style="color: #408080; font-style: italic">#If the feature space is empty, return the mode target feature value of the direct parent node --&gt; Note that</span>
<span style="color: #408080; font-style: italic">#the direct parent node is that node which has called the current run of the ID3 algorithm and hence</span>
<span style="color: #408080; font-style: italic">#the mode target feature value is stored in the parent_node_class variable.</span>
<span style="color: #008000; font-weight: bold">elif</span> <span style="color: #008000">len</span>(features) <span style="color: #666666">==0</span>:
<span style="color: #008000; font-weight: bold">return</span> parent_node_class
<span style="color: #408080; font-style: italic">#If none of the above holds true, grow the tree!</span>
<span style="color: #408080; font-style: italic"># to make this notebook&#39;s output stable across runs</span>
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">42</span>)
<span style="color: #408080; font-style: italic"># To plot pretty figures</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">matplotlib.colors</span> <span style="color: #008000; font-weight: bold">import</span> ListedColormap
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">&#39;axes.labelsize&#39;</span>] <span style="color: #666666">=</span> <span style="color: #666666">14</span>
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">&#39;xtick.labelsize&#39;</span>] <span style="color: #666666">=</span> <span style="color: #666666">12</span>
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">&#39;ytick.labelsize&#39;</span>] <span style="color: #666666">=</span> <span style="color: #666666">12</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_moons
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> export_graphviz
Xm, ym <span style="color: #666666">=</span> make_moons(n_samples<span style="color: #666666">=100</span>, noise<span style="color: #666666">=0.25</span>, random_state<span style="color: #666666">=53</span>)
deep_tree_clf1 <span style="color: #666666">=</span> DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>)
deep_tree_clf2 <span style="color: #666666">=</span> DecisionTreeClassifier(min_samples_leaf<span style="color: #666666">=4</span>, random_state<span style="color: #666666">=42</span>)
deep_tree_clf1<span style="color: #666666">.</span>fit(Xm, ym)
deep_tree_clf2<span style="color: #666666">.</span>fit(Xm, ym)
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_decision_boundary</span>(clf, X, y, axes<span style="color: #666666">=</span>[<span style="color: #666666">0</span>, <span style="color: #666666">7.5</span>, <span style="color: #666666">0</span>, <span style="color: #666666">3</span>], iris<span style="color: #666666">=</span><span style="color: #008000">True</span>, legend<span style="color: #666666">=</span><span style="color: #008000">False</span>, plot_training<span style="color: #666666">=</span><span style="color: #008000">True</span>):
x1s <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">0</span>], axes[<span style="color: #666666">1</span>], <span style="color: #666666">100</span>)
x2s <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">2</span>], axes[<span style="color: #666666">3</span>], <span style="color: #666666">100</span>)
x1, x2 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>meshgrid(x1s, x2s)
X_new <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[x1<span style="color: #666666">.</span>ravel(), x2<span style="color: #666666">.</span>ravel()]
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_new)<span style="color: #666666">.</span>reshape(x1<span style="color: #666666">.</span>shape)
custom_cmap <span style="color: #666666">=</span> ListedColormap([<span style="color: #BA2121">&#39;#fafab0&#39;</span>,<span style="color: #BA2121">&#39;#9898ff&#39;</span>,<span style="color: #BA2121">&#39;#a0faa0&#39;</span>])
plt<span style="color: #666666">.</span>contourf(x1, x2, y_pred, alpha<span style="color: #666666">=0.3</span>, cmap<span style="color: #666666">=</span>custom_cmap)
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> iris:
custom_cmap2 <span style="color: #666666">=</span> ListedColormap([<span style="color: #BA2121">&#39;#7d7d58&#39;</span>,<span style="color: #BA2121">&#39;#4c4c7f&#39;</span>,<span style="color: #BA2121">&#39;#507d50&#39;</span>])
plt<span style="color: #666666">.</span>contour(x1, x2, y_pred, cmap<span style="color: #666666">=</span>custom_cmap2, alpha<span style="color: #666666">=0.8</span>)
<span style="color: #008000; font-weight: bold">if</span> plot_training:
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==0</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==0</span>], <span style="color: #BA2121">&quot;yo&quot;</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;Iris-Setosa&quot;</span>)
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==1</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==1</span>], <span style="color: #BA2121">&quot;bs&quot;</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;Iris-Versicolor&quot;</span>)
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==2</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==2</span>], <span style="color: #BA2121">&quot;g^&quot;</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;Iris-Virginica&quot;</span>)
plt<span style="color: #666666">.</span>axis(axes)
<span style="color: #008000; font-weight: bold">if</span> iris:
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&quot;Petal length&quot;</span>, fontsize<span style="color: #666666">=14</span>)
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">&quot;Petal width&quot;</span>, fontsize<span style="color: #666666">=14</span>)
<span style="color: #008000; font-weight: bold">else</span>:
<span style="color: #408080; font-style: italic">#Set the default value for this node --&gt; The mode target feature value of the current node</span>
parent_node_class <span style="color: #666666">=</span> np<span style="color: #666666">.</span>unique(data[target_attribute_name])[np<span style="color: #666666">.</span>argmax(np<span style="color: #666666">.</span>unique(data[target_attribute_name],return_counts<span style="color: #666666">=</span><span style="color: #008000">True</span>)[<span style="color: #666666">1</span>])]
<span style="color: #408080; font-style: italic">#Select the feature which best splits the dataset</span>
item_values <span style="color: #666666">=</span> [InfoGain(data,feature,target_attribute_name) <span style="color: #008000; font-weight: bold">for</span> feature <span style="color: #AA22FF; font-weight: bold">in</span> features] <span style="color: #408080; font-style: italic">#Return the information gain values for the features in the dataset</span>
best_feature_index <span style="color: #666666">=</span> np<span style="color: #666666">.</span>argmax(item_values)
best_feature <span style="color: #666666">=</span> features[best_feature_index]
<span style="color: #408080; font-style: italic">#Create the tree structure. The root gets the name of the feature (best_feature) with the maximum information</span>
<span style="color: #408080; font-style: italic">#gain in the first run</span>
tree <span style="color: #666666">=</span> {best_feature:{}}
<span style="color: #408080; font-style: italic">#Remove the feature with the best inforamtion gain from the feature space</span>
features <span style="color: #666666">=</span> [i <span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> features <span style="color: #008000; font-weight: bold">if</span> i <span style="color: #666666">!=</span> best_feature]
<span style="color: #408080; font-style: italic">#Grow a branch under the root node for each possible value of the root node feature</span>
<span style="color: #008000; font-weight: bold">for</span> value <span style="color: #AA22FF; font-weight: bold">in</span> np<span style="color: #666666">.</span>unique(data[best_feature]):
value <span style="color: #666666">=</span> value
<span style="color: #408080; font-style: italic">#Split the dataset along the value of the feature with the largest information gain and therwith create sub_datasets</span>
sub_data <span style="color: #666666">=</span> data<span style="color: #666666">.</span>where(data[best_feature] <span style="color: #666666">==</span> value)<span style="color: #666666">.</span>dropna()
<span style="color: #408080; font-style: italic">#Call the ID3 algorithm for each of those sub_datasets with the new parameters --&gt; Here the recursion comes in!</span>
subtree <span style="color: #666666">=</span> ID3(sub_data,dataset,features,target_attribute_name,parent_node_class)
<span style="color: #408080; font-style: italic">#Add the sub tree, grown from the sub_dataset to the tree under the root node</span>
tree[best_feature][value] <span style="color: #666666">=</span> subtree
<span style="color: #008000; font-weight: bold">return</span>(tree)
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r&quot;$x_1$&quot;</span>, fontsize<span style="color: #666666">=18</span>)
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r&quot;$x_2$&quot;</span>, fontsize<span style="color: #666666">=18</span>, rotation<span style="color: #666666">=0</span>)
<span style="color: #008000; font-weight: bold">if</span> legend:
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">&quot;lower right&quot;</span>, fontsize<span style="color: #666666">=14</span>)
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>, <span style="color: #666666">4</span>))
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
plot_decision_boundary(deep_tree_clf1, Xm, ym, axes<span style="color: #666666">=</span>[<span style="color: #666666">-1.5</span>, <span style="color: #666666">2.5</span>, <span style="color: #666666">-1</span>, <span style="color: #666666">1.5</span>], iris<span style="color: #666666">=</span><span style="color: #008000">False</span>)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;No restrictions&quot;</span>, fontsize<span style="color: #666666">=16</span>)
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
plot_decision_boundary(deep_tree_clf2, Xm, ym, axes<span style="color: #666666">=</span>[<span style="color: #666666">-1.5</span>, <span style="color: #666666">2.5</span>, <span style="color: #666666">-1</span>, <span style="color: #666666">1.5</span>], iris<span style="color: #666666">=</span><span style="color: #008000">False</span>)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;min_samples_leaf = {}&quot;</span><span style="color: #666666">.</span>format(deep_tree_clf2<span style="color: #666666">.</span>min_samples_leaf), fontsize<span style="color: #666666">=14</span>)
plt<span style="color: #666666">.</span>show()
</pre></div>
<p>
<p>
@@ -274,7 +252,7 @@ MathJax.Hub.Config({
<li><a href="._DecisionTrees-bs023.html">24</a></li>
<li><a href="._DecisionTrees-bs024.html">25</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs016.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs016.html
+67 -113
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,73 +157,29 @@ MathJax.Hub.Config({
<a name="part0016"></a>
<!-- !split -->
<h2 id="___sec15" class="anchor">Back to moons again </h2>
<h2 id="___sec15" class="anchor">Playing around with regions </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">__future__</span> <span style="color: #008000; font-weight: bold">import</span> division, print_function, unicode_literals
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">6</span>)
Xs <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>, <span style="color: #666666">2</span>) <span style="color: #666666">-</span> <span style="color: #666666">0.5</span>
ys <span style="color: #666666">=</span> (Xs[:, <span style="color: #666666">0</span>] <span style="color: #666666">&gt;</span> <span style="color: #666666">0</span>)<span style="color: #666666">.</span>astype(np<span style="color: #666666">.</span>float32) <span style="color: #666666">*</span> <span style="color: #666666">2</span>
<span style="color: #408080; font-style: italic"># Common imports</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
angle <span style="color: #666666">=</span> np<span style="color: #666666">.</span>pi <span style="color: #666666">/</span> <span style="color: #666666">4</span>
rotation_matrix <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[np<span style="color: #666666">.</span>cos(angle), <span style="color: #666666">-</span>np<span style="color: #666666">.</span>sin(angle)], [np<span style="color: #666666">.</span>sin(angle), np<span style="color: #666666">.</span>cos(angle)]])
Xsr <span style="color: #666666">=</span> Xs<span style="color: #666666">.</span>dot(rotation_matrix)
<span style="color: #408080; font-style: italic"># to make this notebook&#39;s output stable across runs</span>
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">42</span>)
tree_clf_s <span style="color: #666666">=</span> DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>)
tree_clf_s<span style="color: #666666">.</span>fit(Xs, ys)
tree_clf_sr <span style="color: #666666">=</span> DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>)
tree_clf_sr<span style="color: #666666">.</span>fit(Xsr, ys)
<span style="color: #408080; font-style: italic"># To plot pretty figures</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">matplotlib.colors</span> <span style="color: #008000; font-weight: bold">import</span> ListedColormap
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">&#39;axes.labelsize&#39;</span>] <span style="color: #666666">=</span> <span style="color: #666666">14</span>
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">&#39;xtick.labelsize&#39;</span>] <span style="color: #666666">=</span> <span style="color: #666666">12</span>
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">&#39;ytick.labelsize&#39;</span>] <span style="color: #666666">=</span> <span style="color: #666666">12</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_moons
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> export_graphviz
Xm, ym <span style="color: #666666">=</span> make_moons(n_samples<span style="color: #666666">=100</span>, noise<span style="color: #666666">=0.25</span>, random_state<span style="color: #666666">=53</span>)
deep_tree_clf1 <span style="color: #666666">=</span> DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>)
deep_tree_clf2 <span style="color: #666666">=</span> DecisionTreeClassifier(min_samples_leaf<span style="color: #666666">=4</span>, random_state<span style="color: #666666">=42</span>)
deep_tree_clf1<span style="color: #666666">.</span>fit(Xm, ym)
deep_tree_clf2<span style="color: #666666">.</span>fit(Xm, ym)
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_decision_boundary</span>(clf, X, y, axes<span style="color: #666666">=</span>[<span style="color: #666666">0</span>, <span style="color: #666666">7.5</span>, <span style="color: #666666">0</span>, <span style="color: #666666">3</span>], iris<span style="color: #666666">=</span><span style="color: #008000">True</span>, legend<span style="color: #666666">=</span><span style="color: #008000">False</span>, plot_training<span style="color: #666666">=</span><span style="color: #008000">True</span>):
x1s <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">0</span>], axes[<span style="color: #666666">1</span>], <span style="color: #666666">100</span>)
x2s <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">2</span>], axes[<span style="color: #666666">3</span>], <span style="color: #666666">100</span>)
x1, x2 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>meshgrid(x1s, x2s)
X_new <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[x1<span style="color: #666666">.</span>ravel(), x2<span style="color: #666666">.</span>ravel()]
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_new)<span style="color: #666666">.</span>reshape(x1<span style="color: #666666">.</span>shape)
custom_cmap <span style="color: #666666">=</span> ListedColormap([<span style="color: #BA2121">&#39;#fafab0&#39;</span>,<span style="color: #BA2121">&#39;#9898ff&#39;</span>,<span style="color: #BA2121">&#39;#a0faa0&#39;</span>])
plt<span style="color: #666666">.</span>contourf(x1, x2, y_pred, alpha<span style="color: #666666">=0.3</span>, cmap<span style="color: #666666">=</span>custom_cmap)
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> iris:
custom_cmap2 <span style="color: #666666">=</span> ListedColormap([<span style="color: #BA2121">&#39;#7d7d58&#39;</span>,<span style="color: #BA2121">&#39;#4c4c7f&#39;</span>,<span style="color: #BA2121">&#39;#507d50&#39;</span>])
plt<span style="color: #666666">.</span>contour(x1, x2, y_pred, cmap<span style="color: #666666">=</span>custom_cmap2, alpha<span style="color: #666666">=0.8</span>)
<span style="color: #008000; font-weight: bold">if</span> plot_training:
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==0</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==0</span>], <span style="color: #BA2121">&quot;yo&quot;</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;Iris-Setosa&quot;</span>)
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==1</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==1</span>], <span style="color: #BA2121">&quot;bs&quot;</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;Iris-Versicolor&quot;</span>)
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==2</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==2</span>], <span style="color: #BA2121">&quot;g^&quot;</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;Iris-Virginica&quot;</span>)
plt<span style="color: #666666">.</span>axis(axes)
<span style="color: #008000; font-weight: bold">if</span> iris:
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&quot;Petal length&quot;</span>, fontsize<span style="color: #666666">=14</span>)
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">&quot;Petal width&quot;</span>, fontsize<span style="color: #666666">=14</span>)
<span style="color: #008000; font-weight: bold">else</span>:
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r&quot;$x_1$&quot;</span>, fontsize<span style="color: #666666">=18</span>)
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r&quot;$x_2$&quot;</span>, fontsize<span style="color: #666666">=18</span>, rotation<span style="color: #666666">=0</span>)
<span style="color: #008000; font-weight: bold">if</span> legend:
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">&quot;lower right&quot;</span>, fontsize<span style="color: #666666">=14</span>)
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>, <span style="color: #666666">4</span>))
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
plot_decision_boundary(deep_tree_clf1, Xm, ym, axes<span style="color: #666666">=</span>[<span style="color: #666666">-1.5</span>, <span style="color: #666666">2.5</span>, <span style="color: #666666">-1</span>, <span style="color: #666666">1.5</span>], iris<span style="color: #666666">=</span><span style="color: #008000">False</span>)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;No restrictions&quot;</span>, fontsize<span style="color: #666666">=16</span>)
plot_decision_boundary(tree_clf_s, Xs, ys, axes<span style="color: #666666">=</span>[<span style="color: #666666">-0.7</span>, <span style="color: #666666">0.7</span>, <span style="color: #666666">-0.7</span>, <span style="color: #666666">0.7</span>], iris<span style="color: #666666">=</span><span style="color: #008000">False</span>)
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
plot_decision_boundary(deep_tree_clf2, Xm, ym, axes<span style="color: #666666">=</span>[<span style="color: #666666">-1.5</span>, <span style="color: #666666">2.5</span>, <span style="color: #666666">-1</span>, <span style="color: #666666">1.5</span>], iris<span style="color: #666666">=</span><span style="color: #008000">False</span>)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;min_samples_leaf = {}&quot;</span><span style="color: #666666">.</span>format(deep_tree_clf2<span style="color: #666666">.</span>min_samples_leaf), fontsize<span style="color: #666666">=14</span>)
plot_decision_boundary(tree_clf_sr, Xsr, ys, axes<span style="color: #666666">=</span>[<span style="color: #666666">-0.7</span>, <span style="color: #666666">0.7</span>, <span style="color: #666666">-0.7</span>, <span style="color: #666666">0.7</span>], iris<span style="color: #666666">=</span><span style="color: #008000">False</span>)
plt<span style="color: #666666">.</span>show()
</pre></div>
<p>
@@ -254,7 +208,7 @@ plt<span style="color: #666666">.</span>show()
<li><a href="._DecisionTrees-bs024.html">25</a></li>
<li><a href="._DecisionTrees-bs025.html">26</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs017.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs017.html
+66 -74
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,30 +157,24 @@ MathJax.Hub.Config({
<a name="part0017"></a>
<!-- !split -->
<h2 id="___sec16" class="anchor">Playing around with regions </h2>
<h2 id="___sec16" class="anchor">Regression trees </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">6</span>)
Xs <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>, <span style="color: #666666">2</span>) <span style="color: #666666">-</span> <span style="color: #666666">0.5</span>
ys <span style="color: #666666">=</span> (Xs[:, <span style="color: #666666">0</span>] <span style="color: #666666">&gt;</span> <span style="color: #666666">0</span>)<span style="color: #666666">.</span>astype(np<span style="color: #666666">.</span>float32) <span style="color: #666666">*</span> <span style="color: #666666">2</span>
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Quadratic training set + noise</span>
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">42</span>)
m <span style="color: #666666">=</span> <span style="color: #666666">200</span>
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(m, <span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">4</span> <span style="color: #666666">*</span> (X <span style="color: #666666">-</span> <span style="color: #666666">0.5</span>) <span style="color: #666666">**</span> <span style="color: #666666">2</span>
y <span style="color: #666666">=</span> y <span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(m, <span style="color: #666666">1</span>) <span style="color: #666666">/</span> <span style="color: #666666">10</span>
</pre></div>
<p>
angle <span style="color: #666666">=</span> np<span style="color: #666666">.</span>pi <span style="color: #666666">/</span> <span style="color: #666666">4</span>
rotation_matrix <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[np<span style="color: #666666">.</span>cos(angle), <span style="color: #666666">-</span>np<span style="color: #666666">.</span>sin(angle)], [np<span style="color: #666666">.</span>sin(angle), np<span style="color: #666666">.</span>cos(angle)]])
Xsr <span style="color: #666666">=</span> Xs<span style="color: #666666">.</span>dot(rotation_matrix)
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeRegressor
tree_clf_s <span style="color: #666666">=</span> DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>)
tree_clf_s<span style="color: #666666">.</span>fit(Xs, ys)
tree_clf_sr <span style="color: #666666">=</span> DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>)
tree_clf_sr<span style="color: #666666">.</span>fit(Xsr, ys)
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>, <span style="color: #666666">4</span>))
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
plot_decision_boundary(tree_clf_s, Xs, ys, axes<span style="color: #666666">=</span>[<span style="color: #666666">-0.7</span>, <span style="color: #666666">0.7</span>, <span style="color: #666666">-0.7</span>, <span style="color: #666666">0.7</span>], iris<span style="color: #666666">=</span><span style="color: #008000">False</span>)
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
plot_decision_boundary(tree_clf_sr, Xsr, ys, axes<span style="color: #666666">=</span>[<span style="color: #666666">-0.7</span>, <span style="color: #666666">0.7</span>, <span style="color: #666666">-0.7</span>, <span style="color: #666666">0.7</span>], iris<span style="color: #666666">=</span><span style="color: #008000">False</span>)
plt<span style="color: #666666">.</span>show()
tree_reg <span style="color: #666666">=</span> DecisionTreeRegressor(max_depth<span style="color: #666666">=2</span>, random_state<span style="color: #666666">=42</span>)
tree_reg<span style="color: #666666">.</span>fit(X, y)
</pre></div>
<p>
<p>
@@ -210,7 +202,7 @@ plt<span style="color: #666666">.</span>show()
<li><a href="._DecisionTrees-bs025.html">26</a></li>
<li><a href="._DecisionTrees-bs026.html">27</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs018.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs018.html
+122 -68
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,24 +157,80 @@ MathJax.Hub.Config({
<a name="part0018"></a>
<!-- !split -->
<h2 id="___sec17" class="anchor">Regression trees </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Quadratic training set + noise</span>
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">42</span>)
m <span style="color: #666666">=</span> <span style="color: #666666">200</span>
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(m, <span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">4</span> <span style="color: #666666">*</span> (X <span style="color: #666666">-</span> <span style="color: #666666">0.5</span>) <span style="color: #666666">**</span> <span style="color: #666666">2</span>
y <span style="color: #666666">=</span> y <span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(m, <span style="color: #666666">1</span>) <span style="color: #666666">/</span> <span style="color: #666666">10</span>
</pre></div>
<h2 id="___sec17" class="anchor">Final regressor code </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeRegressor
tree_reg <span style="color: #666666">=</span> DecisionTreeRegressor(max_depth<span style="color: #666666">=2</span>, random_state<span style="color: #666666">=42</span>)
tree_reg<span style="color: #666666">.</span>fit(X, y)
tree_reg1 <span style="color: #666666">=</span> DecisionTreeRegressor(random_state<span style="color: #666666">=42</span>, max_depth<span style="color: #666666">=2</span>)
tree_reg2 <span style="color: #666666">=</span> DecisionTreeRegressor(random_state<span style="color: #666666">=42</span>, max_depth<span style="color: #666666">=3</span>)
tree_reg1<span style="color: #666666">.</span>fit(X, y)
tree_reg2<span style="color: #666666">.</span>fit(X, y)
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_regression_predictions</span>(tree_reg, X, y, axes<span style="color: #666666">=</span>[<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">-0.2</span>, <span style="color: #666666">1</span>], ylabel<span style="color: #666666">=</span><span style="color: #BA2121">&quot;$y$&quot;</span>):
x1 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">0</span>], axes[<span style="color: #666666">1</span>], <span style="color: #666666">500</span>)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
y_pred <span style="color: #666666">=</span> tree_reg<span style="color: #666666">.</span>predict(x1)
plt<span style="color: #666666">.</span>axis(axes)
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&quot;$x_1$&quot;</span>, fontsize<span style="color: #666666">=18</span>)
<span style="color: #008000; font-weight: bold">if</span> ylabel:
plt<span style="color: #666666">.</span>ylabel(ylabel, fontsize<span style="color: #666666">=18</span>, rotation<span style="color: #666666">=0</span>)
plt<span style="color: #666666">.</span>plot(X, y, <span style="color: #BA2121">&quot;b.&quot;</span>)
plt<span style="color: #666666">.</span>plot(x1, y_pred, <span style="color: #BA2121">&quot;r.-&quot;</span>, linewidth<span style="color: #666666">=2</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">r&quot;$\hat{y}$&quot;</span>)
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>, <span style="color: #666666">4</span>))
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
plot_regression_predictions(tree_reg1, X, y)
<span style="color: #008000; font-weight: bold">for</span> split, style <span style="color: #AA22FF; font-weight: bold">in</span> ((<span style="color: #666666">0.1973</span>, <span style="color: #BA2121">&quot;k-&quot;</span>), (<span style="color: #666666">0.0917</span>, <span style="color: #BA2121">&quot;k--&quot;</span>), (<span style="color: #666666">0.7718</span>, <span style="color: #BA2121">&quot;k--&quot;</span>)):
plt<span style="color: #666666">.</span>plot([split, split], [<span style="color: #666666">-0.2</span>, <span style="color: #666666">1</span>], style, linewidth<span style="color: #666666">=2</span>)
plt<span style="color: #666666">.</span>text(<span style="color: #666666">0.21</span>, <span style="color: #666666">0.65</span>, <span style="color: #BA2121">&quot;Depth=0&quot;</span>, fontsize<span style="color: #666666">=15</span>)
plt<span style="color: #666666">.</span>text(<span style="color: #666666">0.01</span>, <span style="color: #666666">0.2</span>, <span style="color: #BA2121">&quot;Depth=1&quot;</span>, fontsize<span style="color: #666666">=13</span>)
plt<span style="color: #666666">.</span>text(<span style="color: #666666">0.65</span>, <span style="color: #666666">0.8</span>, <span style="color: #BA2121">&quot;Depth=1&quot;</span>, fontsize<span style="color: #666666">=13</span>)
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">&quot;upper center&quot;</span>, fontsize<span style="color: #666666">=18</span>)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;max_depth=2&quot;</span>, fontsize<span style="color: #666666">=14</span>)
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
plot_regression_predictions(tree_reg2, X, y, ylabel<span style="color: #666666">=</span><span style="color: #008000">None</span>)
<span style="color: #008000; font-weight: bold">for</span> split, style <span style="color: #AA22FF; font-weight: bold">in</span> ((<span style="color: #666666">0.1973</span>, <span style="color: #BA2121">&quot;k-&quot;</span>), (<span style="color: #666666">0.0917</span>, <span style="color: #BA2121">&quot;k--&quot;</span>), (<span style="color: #666666">0.7718</span>, <span style="color: #BA2121">&quot;k--&quot;</span>)):
plt<span style="color: #666666">.</span>plot([split, split], [<span style="color: #666666">-0.2</span>, <span style="color: #666666">1</span>], style, linewidth<span style="color: #666666">=2</span>)
<span style="color: #008000; font-weight: bold">for</span> split <span style="color: #AA22FF; font-weight: bold">in</span> (<span style="color: #666666">0.0458</span>, <span style="color: #666666">0.1298</span>, <span style="color: #666666">0.2873</span>, <span style="color: #666666">0.9040</span>):
plt<span style="color: #666666">.</span>plot([split, split], [<span style="color: #666666">-0.2</span>, <span style="color: #666666">1</span>], <span style="color: #BA2121">&quot;k:&quot;</span>, linewidth<span style="color: #666666">=1</span>)
plt<span style="color: #666666">.</span>text(<span style="color: #666666">0.3</span>, <span style="color: #666666">0.5</span>, <span style="color: #BA2121">&quot;Depth=2&quot;</span>, fontsize<span style="color: #666666">=13</span>)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;max_depth=3&quot;</span>, fontsize<span style="color: #666666">=14</span>)
plt<span style="color: #666666">.</span>show()
</pre></div>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>tree_reg1 <span style="color: #666666">=</span> DecisionTreeRegressor(random_state<span style="color: #666666">=42</span>)
tree_reg2 <span style="color: #666666">=</span> DecisionTreeRegressor(random_state<span style="color: #666666">=42</span>, min_samples_leaf<span style="color: #666666">=10</span>)
tree_reg1<span style="color: #666666">.</span>fit(X, y)
tree_reg2<span style="color: #666666">.</span>fit(X, y)
x1 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">500</span>)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
y_pred1 <span style="color: #666666">=</span> tree_reg1<span style="color: #666666">.</span>predict(x1)
y_pred2 <span style="color: #666666">=</span> tree_reg2<span style="color: #666666">.</span>predict(x1)
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>, <span style="color: #666666">4</span>))
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
plt<span style="color: #666666">.</span>plot(X, y, <span style="color: #BA2121">&quot;b.&quot;</span>)
plt<span style="color: #666666">.</span>plot(x1, y_pred1, <span style="color: #BA2121">&quot;r.-&quot;</span>, linewidth<span style="color: #666666">=2</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">r&quot;$\hat{y}$&quot;</span>)
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">-0.2</span>, <span style="color: #666666">1.1</span>])
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&quot;$x_1$&quot;</span>, fontsize<span style="color: #666666">=18</span>)
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">&quot;$y$&quot;</span>, fontsize<span style="color: #666666">=18</span>, rotation<span style="color: #666666">=0</span>)
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">&quot;upper center&quot;</span>, fontsize<span style="color: #666666">=18</span>)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;No restrictions&quot;</span>, fontsize<span style="color: #666666">=14</span>)
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
plt<span style="color: #666666">.</span>plot(X, y, <span style="color: #BA2121">&quot;b.&quot;</span>)
plt<span style="color: #666666">.</span>plot(x1, y_pred2, <span style="color: #BA2121">&quot;r.-&quot;</span>, linewidth<span style="color: #666666">=2</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">r&quot;$\hat{y}$&quot;</span>)
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">-0.2</span>, <span style="color: #666666">1.1</span>])
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&quot;$x_1$&quot;</span>, fontsize<span style="color: #666666">=18</span>)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;min_samples_leaf={}&quot;</span><span style="color: #666666">.</span>format(tree_reg2<span style="color: #666666">.</span>min_samples_leaf), fontsize<span style="color: #666666">=14</span>)
plt<span style="color: #666666">.</span>show()
</pre></div>
<p>
<p>
@@ -204,7 +258,7 @@ tree_reg<span style="color: #666666">.</span>fit(X, y)
<li><a href="._DecisionTrees-bs026.html">27</a></li>
<li><a href="._DecisionTrees-bs027.html">28</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs019.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs019.html
+73 -125
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,80 +157,30 @@ MathJax.Hub.Config({
<a name="part0019"></a>
<!-- !split -->
<h2 id="___sec18" class="anchor">Final regressor code </h2>
<h2 id="___sec18" class="anchor">Classification again: The zoo data </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeRegressor
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">pprint</span> <span style="color: #008000; font-weight: bold">import</span> pprint
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
tree_reg1 <span style="color: #666666">=</span> DecisionTreeRegressor(random_state<span style="color: #666666">=42</span>, max_depth<span style="color: #666666">=2</span>)
tree_reg2 <span style="color: #666666">=</span> DecisionTreeRegressor(random_state<span style="color: #666666">=42</span>, max_depth<span style="color: #666666">=3</span>)
tree_reg1<span style="color: #666666">.</span>fit(X, y)
tree_reg2<span style="color: #666666">.</span>fit(X, y)
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_regression_predictions</span>(tree_reg, X, y, axes<span style="color: #666666">=</span>[<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">-0.2</span>, <span style="color: #666666">1</span>], ylabel<span style="color: #666666">=</span><span style="color: #BA2121">&quot;$y$&quot;</span>):
x1 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">0</span>], axes[<span style="color: #666666">1</span>], <span style="color: #666666">500</span>)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
y_pred <span style="color: #666666">=</span> tree_reg<span style="color: #666666">.</span>predict(x1)
plt<span style="color: #666666">.</span>axis(axes)
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&quot;$x_1$&quot;</span>, fontsize<span style="color: #666666">=18</span>)
<span style="color: #008000; font-weight: bold">if</span> ylabel:
plt<span style="color: #666666">.</span>ylabel(ylabel, fontsize<span style="color: #666666">=18</span>, rotation<span style="color: #666666">=0</span>)
plt<span style="color: #666666">.</span>plot(X, y, <span style="color: #BA2121">&quot;b.&quot;</span>)
plt<span style="color: #666666">.</span>plot(x1, y_pred, <span style="color: #BA2121">&quot;r.-&quot;</span>, linewidth<span style="color: #666666">=2</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">r&quot;$\hat{y}$&quot;</span>)
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>, <span style="color: #666666">4</span>))
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
plot_regression_predictions(tree_reg1, X, y)
<span style="color: #008000; font-weight: bold">for</span> split, style <span style="color: #AA22FF; font-weight: bold">in</span> ((<span style="color: #666666">0.1973</span>, <span style="color: #BA2121">&quot;k-&quot;</span>), (<span style="color: #666666">0.0917</span>, <span style="color: #BA2121">&quot;k--&quot;</span>), (<span style="color: #666666">0.7718</span>, <span style="color: #BA2121">&quot;k--&quot;</span>)):
plt<span style="color: #666666">.</span>plot([split, split], [<span style="color: #666666">-0.2</span>, <span style="color: #666666">1</span>], style, linewidth<span style="color: #666666">=2</span>)
plt<span style="color: #666666">.</span>text(<span style="color: #666666">0.21</span>, <span style="color: #666666">0.65</span>, <span style="color: #BA2121">&quot;Depth=0&quot;</span>, fontsize<span style="color: #666666">=15</span>)
plt<span style="color: #666666">.</span>text(<span style="color: #666666">0.01</span>, <span style="color: #666666">0.2</span>, <span style="color: #BA2121">&quot;Depth=1&quot;</span>, fontsize<span style="color: #666666">=13</span>)
plt<span style="color: #666666">.</span>text(<span style="color: #666666">0.65</span>, <span style="color: #666666">0.8</span>, <span style="color: #BA2121">&quot;Depth=1&quot;</span>, fontsize<span style="color: #666666">=13</span>)
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">&quot;upper center&quot;</span>, fontsize<span style="color: #666666">=18</span>)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;max_depth=2&quot;</span>, fontsize<span style="color: #666666">=14</span>)
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
plot_regression_predictions(tree_reg2, X, y, ylabel<span style="color: #666666">=</span><span style="color: #008000">None</span>)
<span style="color: #008000; font-weight: bold">for</span> split, style <span style="color: #AA22FF; font-weight: bold">in</span> ((<span style="color: #666666">0.1973</span>, <span style="color: #BA2121">&quot;k-&quot;</span>), (<span style="color: #666666">0.0917</span>, <span style="color: #BA2121">&quot;k--&quot;</span>), (<span style="color: #666666">0.7718</span>, <span style="color: #BA2121">&quot;k--&quot;</span>)):
plt<span style="color: #666666">.</span>plot([split, split], [<span style="color: #666666">-0.2</span>, <span style="color: #666666">1</span>], style, linewidth<span style="color: #666666">=2</span>)
<span style="color: #008000; font-weight: bold">for</span> split <span style="color: #AA22FF; font-weight: bold">in</span> (<span style="color: #666666">0.0458</span>, <span style="color: #666666">0.1298</span>, <span style="color: #666666">0.2873</span>, <span style="color: #666666">0.9040</span>):
plt<span style="color: #666666">.</span>plot([split, split], [<span style="color: #666666">-0.2</span>, <span style="color: #666666">1</span>], <span style="color: #BA2121">&quot;k:&quot;</span>, linewidth<span style="color: #666666">=1</span>)
plt<span style="color: #666666">.</span>text(<span style="color: #666666">0.3</span>, <span style="color: #666666">0.5</span>, <span style="color: #BA2121">&quot;Depth=2&quot;</span>, fontsize<span style="color: #666666">=13</span>)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;max_depth=3&quot;</span>, fontsize<span style="color: #666666">=14</span>)
plt<span style="color: #666666">.</span>show()
</pre></div>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>tree_reg1 <span style="color: #666666">=</span> DecisionTreeRegressor(random_state<span style="color: #666666">=42</span>)
tree_reg2 <span style="color: #666666">=</span> DecisionTreeRegressor(random_state<span style="color: #666666">=42</span>, min_samples_leaf<span style="color: #666666">=10</span>)
tree_reg1<span style="color: #666666">.</span>fit(X, y)
tree_reg2<span style="color: #666666">.</span>fit(X, y)
x1 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">500</span>)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
y_pred1 <span style="color: #666666">=</span> tree_reg1<span style="color: #666666">.</span>predict(x1)
y_pred2 <span style="color: #666666">=</span> tree_reg2<span style="color: #666666">.</span>predict(x1)
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>, <span style="color: #666666">4</span>))
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
plt<span style="color: #666666">.</span>plot(X, y, <span style="color: #BA2121">&quot;b.&quot;</span>)
plt<span style="color: #666666">.</span>plot(x1, y_pred1, <span style="color: #BA2121">&quot;r.-&quot;</span>, linewidth<span style="color: #666666">=2</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">r&quot;$\hat{y}$&quot;</span>)
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">-0.2</span>, <span style="color: #666666">1.1</span>])
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&quot;$x_1$&quot;</span>, fontsize<span style="color: #666666">=18</span>)
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">&quot;$y$&quot;</span>, fontsize<span style="color: #666666">=18</span>, rotation<span style="color: #666666">=0</span>)
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">&quot;upper center&quot;</span>, fontsize<span style="color: #666666">=18</span>)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;No restrictions&quot;</span>, fontsize<span style="color: #666666">=14</span>)
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
plt<span style="color: #666666">.</span>plot(X, y, <span style="color: #BA2121">&quot;b.&quot;</span>)
plt<span style="color: #666666">.</span>plot(x1, y_pred2, <span style="color: #BA2121">&quot;r.-&quot;</span>, linewidth<span style="color: #666666">=2</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">r&quot;$\hat{y}$&quot;</span>)
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">-0.2</span>, <span style="color: #666666">1.1</span>])
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&quot;$x_1$&quot;</span>, fontsize<span style="color: #666666">=18</span>)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;min_samples_leaf={}&quot;</span><span style="color: #666666">.</span>format(tree_reg2<span style="color: #666666">.</span>min_samples_leaf), fontsize<span style="color: #666666">=14</span>)
plt<span style="color: #666666">.</span>show()
<span style="color: #408080; font-style: italic">#Import the dataset </span>
dataset <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>read_csv(<span style="color: #BA2121">&#39;data/zoo.csv&#39;</span>)
<span style="color: #408080; font-style: italic">#We drop the animal names since this is not a good feature to split the data on</span>
<span style="color: #408080; font-style: italic">#dataset=dataset.drop(&#39;animal_name&#39;,axis=1)</span>
<span style="color: #408080; font-style: italic">#Split the data into a training and a testing set</span>
train_features <span style="color: #666666">=</span> dataset<span style="color: #666666">.</span>iloc[:<span style="color: #666666">80</span>,:<span style="color: #666666">-1</span>]
test_features <span style="color: #666666">=</span> dataset<span style="color: #666666">.</span>iloc[<span style="color: #666666">80</span>:,:<span style="color: #666666">-1</span>]
train_targets <span style="color: #666666">=</span> dataset<span style="color: #666666">.</span>iloc[:<span style="color: #666666">80</span>,<span style="color: #666666">-1</span>]
test_targets <span style="color: #666666">=</span> dataset<span style="color: #666666">.</span>iloc[<span style="color: #666666">80</span>:,<span style="color: #666666">-1</span>]
<span style="color: #408080; font-style: italic">#Train the model</span>
tree <span style="color: #666666">=</span> DecisionTreeClassifier(criterion <span style="color: #666666">=</span> <span style="color: #BA2121">&#39;entropy&#39;</span>)<span style="color: #666666">.</span>fit(train_features,train_targets)
<span style="color: #408080; font-style: italic">#Predict the classes of new, unseen data</span>
prediction <span style="color: #666666">=</span> tree<span style="color: #666666">.</span>predict(test_features)
<span style="color: #408080; font-style: italic">#Check the accuracy</span>
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">&quot;The prediction accuracy is: &quot;</span>,tree<span style="color: #666666">.</span>score(test_features,test_targets)<span style="color: #666666">*100</span>,<span style="color: #BA2121">&quot;%&quot;</span>)
</pre></div>
<p>
<p>
@@ -260,7 +208,7 @@ plt<span style="color: #666666">.</span>show()
<li><a href="._DecisionTrees-bs027.html">28</a></li>
<li><a href="._DecisionTrees-bs028.html">29</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs020.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs020.html
+62 -80
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,32 +157,18 @@ MathJax.Hub.Config({
<a name="part0020"></a>
<!-- !split -->
<h2 id="___sec19" class="anchor">Classification again: The zoo data </h2>
<p>
<h2 id="___sec19" class="anchor">Pros and cons of trees, pros </h2>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">pprint</span> <span style="color: #008000; font-weight: bold">import</span> pprint
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
<ul>
<li> White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines)</li>
<li> Trees are very easy to explain to people. In fact, they are even easier to explain than linear regression!</li>
<li> No feature normalization needed</li>
<li> Tree models can handle both continuous and categorical data (Classification and Regression Trees)</li>
<li> Can model nonlinear relationships</li>
<li> Can model interactions between the different descriptive features</li>
<li> Trees can be displayed graphically, and are easily interpreted even by a non-expert (especially if they are small)</li>
</ul>
<span style="color: #408080; font-style: italic">#Import the dataset </span>
dataset <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>read_csv(<span style="color: #BA2121">&#39;data/zoo.csv&#39;</span>)
<span style="color: #408080; font-style: italic">#We drop the animal names since this is not a good feature to split the data on</span>
<span style="color: #408080; font-style: italic">#dataset=dataset.drop(&#39;animal_name&#39;,axis=1)</span>
<span style="color: #408080; font-style: italic">#Split the data into a training and a testing set</span>
train_features <span style="color: #666666">=</span> dataset<span style="color: #666666">.</span>iloc[:<span style="color: #666666">80</span>,:<span style="color: #666666">-1</span>]
test_features <span style="color: #666666">=</span> dataset<span style="color: #666666">.</span>iloc[<span style="color: #666666">80</span>:,:<span style="color: #666666">-1</span>]
train_targets <span style="color: #666666">=</span> dataset<span style="color: #666666">.</span>iloc[:<span style="color: #666666">80</span>,<span style="color: #666666">-1</span>]
test_targets <span style="color: #666666">=</span> dataset<span style="color: #666666">.</span>iloc[<span style="color: #666666">80</span>:,<span style="color: #666666">-1</span>]
<span style="color: #408080; font-style: italic">#Train the model</span>
tree <span style="color: #666666">=</span> DecisionTreeClassifier(criterion <span style="color: #666666">=</span> <span style="color: #BA2121">&#39;entropy&#39;</span>)<span style="color: #666666">.</span>fit(train_features,train_targets)
<span style="color: #408080; font-style: italic">#Predict the classes of new, unseen data</span>
prediction <span style="color: #666666">=</span> tree<span style="color: #666666">.</span>predict(test_features)
<span style="color: #408080; font-style: italic">#Check the accuracy</span>
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">&quot;The prediction accuracy is: &quot;</span>,tree<span style="color: #666666">.</span>score(test_features,test_targets)<span style="color: #666666">*100</span>,<span style="color: #BA2121">&quot;%&quot;</span>)
</pre></div>
<p>
<p>
<!-- navigation buttons at the bottom of the page -->
<ul class="pagination">
@@ -209,8 +193,6 @@ prediction <span style="color: #666666">=</span> tree<span style="color: #666666
<li><a href="._DecisionTrees-bs027.html">28</a></li>
<li><a href="._DecisionTrees-bs028.html">29</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs021.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs021.html
+63 -63
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,18 +157,21 @@ MathJax.Hub.Config({
<a name="part0021"></a>
<!-- !split -->
<h2 id="___sec20" class="anchor">Pros and cons of trees, pros </h2>
<h2 id="___sec20" class="anchor">Disadvantages </h2>
<ul>
<li> White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines)</li>
<li> Trees are very easy to explain to people. In fact, they are even easier to explain than linear regression!</li>
<li> No feature normalization needed</li>
<li> Tree models can handle both continuous and categorical data (Classification and Regression Trees)</li>
<li> Can model nonlinear relationships</li>
<li> Can model interactions between the different descriptive features</li>
<li> Trees can be displayed graphically, and are easily interpreted even by a non-expert (especially if they are small)</li>
<li> Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches</li>
<li> If continuous features are used the tree may become quite large and hence less interpretable</li>
<li> Decision trees are prone to overfit the training data and hence do not well generalize the data if no stopping criteria or improvements like pruning, boosting or bagging are implemented</li>
<li> Small changes in the data may lead to a completely different tree. This issue can be addressed by using ensemble methods like bagging, boosting or random forests</li>
<li> Unbalanced datasets where some target feature values occur much more frequently than others may lead to biased trees since the frequently occurring feature values are preferred over the less frequently occurring ones.</li>
<li> If the number of features is relatively large (high dimensional) and the number of instances is relatively low, the tree might overfit the data</li>
<li> Features with many levels may be preferred over features with less levels since for them it is <em>more easy</em> to split the dataset such that the sub datasets only contain pure target feature values. This issue can be addressed by preferring for instance the information gain ratio as splitting criteria over information gain</li>
</ul>
However, by aggregating many decision trees, using methods like bagging, random forests, and boosting, the predictive performance of trees can be substantially improved.
<p>
<p>
<!-- navigation buttons at the bottom of the page -->
<ul class="pagination">
@@ -194,7 +195,6 @@ MathJax.Hub.Config({
<li><a href="._DecisionTrees-bs027.html">28</a></li>
<li><a href="._DecisionTrees-bs028.html">29</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs022.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs022.html
+89 -66
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,19 +157,45 @@ MathJax.Hub.Config({
<a name="part0022"></a>
<!-- !split -->
<h2 id="___sec21" class="anchor">Disadvantages </h2>
<h2 id="___sec21" class="anchor">Bagging </h2>
<ul>
<li> Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches</li>
<li> If continuous features are used the tree may become quite large and hence less interpretable</li>
<li> Decision trees are prone to overfit the training data and hence do not well generalize the data if no stopping criteria or improvements like pruning, boosting or bagging are implemented</li>
<li> Small changes in the data may lead to a completely different tree. This issue can be addressed by using ensemble methods like bagging, boosting or random forests</li>
<li> Unbalanced datasets where some target feature values occur much more frequently than others may lead to biased trees since the frequently occurring feature values are preferred over the less frequently occurring ones.</li>
<li> If the number of features is relatively large (high dimensional) and the number of instances is relatively low, the tree might overfit the data</li>
<li> Features with many levels may be preferred over features with less levels since for them it is <em>more easy</em> to split the dataset such that the sub datasets only contain pure target feature values. This issue can be addressed by preferring for instance the information gain ratio as splitting criteria over information gain</li>
</ul>
<p>
The <b>plain</b> decision trees suffer from high
variance. This means that if we split the training data into two parts
at random, and fit a decision tree to both halves, the results that we
get could be quite different. In contrast, a procedure with low
variance will yield similar results if applied repeatedly to distinct
data sets; linear regression tends to have low variance, if the ratio
of \( n \) to \( p \) is moderately large.
However, by aggregating many decision trees, using methods like bagging, random forests, and boosting, the predictive performance of trees can be substantially improved.
<p>
<b>Bootstrap aggregation</b>, or just <b>bagging</b>, is a
general-purpose procedure for reducing the variance of a statistical
learning method.
<p>
Bagging typically results in improved accuracy
over prediction using a single tree. Unfortunately, however, it can be
difficult to interpret the resulting model. Recall that one of the
advantages of decision trees is the attractive and easily interpreted
diagram that results.
<p>
However, when we bag a large number of trees, it is no longer
possible to represent the resulting statistical learning procedure
using a single tree, and it is no longer clear which variables are
most important to the procedure. Thus, bagging improves prediction
accuracy at the expense of interpretability. Although the collection
of bagged trees is much more difficult to interpret than a single
tree, one can obtain an overall summary of the importance of each
predictor using the MSE (for bagging regression trees) or the Gini
index (for bagging classification trees). In the case of bagging
regression trees, we can record the total amount that the MSE is
decreased due to splits over a given predictor, averaged over all \( B \) possible
trees. A large value indicates an important predictor. Similarly, in
the context of bagging classification trees, we can add up the total
amount that the Gini index is decreased by splits over a given
predictor, averaged over all \( B \) trees.
<p>
<p>
@@ -196,7 +220,6 @@ However, by aggregating many decision trees, using methods like bagging, random
<li><a href="._DecisionTrees-bs027.html">28</a></li>
<li><a href="._DecisionTrees-bs028.html">29</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs023.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs023.html
+67 -93
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,46 +157,23 @@ MathJax.Hub.Config({
<a name="part0023"></a>
<!-- !split -->
<h2 id="___sec22" class="anchor">Bagging </h2>
<h2 id="___sec22" class="anchor">Simple example, head or tail </h2>
<p>
The <b>plain</b> decision trees suffer from high
variance. This means that if we split the training data into two parts
at random, and fit a decision tree to both halves, the results that we
get could be quite different. In contrast, a procedure with low
variance will yield similar results if applied repeatedly to distinct
data sets; linear regression tends to have low variance, if the ratio
of \( n \) to \( p \) is moderately large.
<p>
<b>Bootstrap aggregation</b>, or just <b>bagging</b>, is a
general-purpose procedure for reducing the variance of a statistical
learning method.
<p>
Bagging typically results in improved accuracy
over prediction using a single tree. Unfortunately, however, it can be
difficult to interpret the resulting model. Recall that one of the
advantages of decision trees is the attractive and easily interpreted
diagram that results.
<p>
However, when we bag a large number of trees, it is no longer
possible to represent the resulting statistical learning procedure
using a single tree, and it is no longer clear which variables are
most important to the procedure. Thus, bagging improves prediction
accuracy at the expense of interpretability. Although the collection
of bagged trees is much more difficult to interpret than a single
tree, one can obtain an overall summary of the importance of each
predictor using the MSE (for bagging regression trees) or the Gini
index (for bagging classification trees). In the case of bagging
regression trees, we can record the total amount that the MSE is
decreased due to splits over a given predictor, averaged over all \( B \) possible
trees. A large value indicates an important predictor. Similarly, in
the context of bagging classification trees, we can add up the total
amount that the Gini index is decreased by splits over a given
predictor, averaged over all \( B \) trees.
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>heads_proba <span style="color: #666666">=</span> <span style="color: #666666">0.51</span>
coin_tosses <span style="color: #666666">=</span> (np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">10000</span>, <span style="color: #666666">10</span>) <span style="color: #666666">&lt;</span> heads_proba)<span style="color: #666666">.</span>astype(np<span style="color: #666666">.</span>int32)
cumulative_heads_ratio <span style="color: #666666">=</span> np<span style="color: #666666">.</span>cumsum(coin_tosses, axis<span style="color: #666666">=0</span>) <span style="color: #666666">/</span> np<span style="color: #666666">.</span>arange(<span style="color: #666666">1</span>, <span style="color: #666666">10001</span>)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">8</span>,<span style="color: #666666">3.5</span>))
plt<span style="color: #666666">.</span>plot(cumulative_heads_ratio)
plt<span style="color: #666666">.</span>plot([<span style="color: #666666">0</span>, <span style="color: #666666">10000</span>], [<span style="color: #666666">0.51</span>, <span style="color: #666666">0.51</span>], <span style="color: #BA2121">&quot;k--&quot;</span>, linewidth<span style="color: #666666">=2</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;51%&quot;</span>)
plt<span style="color: #666666">.</span>plot([<span style="color: #666666">0</span>, <span style="color: #666666">10000</span>], [<span style="color: #666666">0.5</span>, <span style="color: #666666">0.5</span>], <span style="color: #BA2121">&quot;k-&quot;</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;50%&quot;</span>)
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&quot;Number of coin tosses&quot;</span>)
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">&quot;Heads ratio&quot;</span>)
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">&quot;lower right&quot;</span>)
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>, <span style="color: #666666">10000</span>, <span style="color: #666666">0.42</span>, <span style="color: #666666">0.58</span>])
plt<span style="color: #666666">.</span>show()
</pre></div>
<p>
<p>
<!-- navigation buttons at the bottom of the page -->
@@ -221,7 +196,6 @@ predictor, averaged over all \( B \) trees.
<li><a href="._DecisionTrees-bs027.html">28</a></li>
<li><a href="._DecisionTrees-bs028.html">29</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs024.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs024.html
+90 -71
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,23 +157,45 @@ MathJax.Hub.Config({
<a name="part0024"></a>
<!-- !split -->
<h2 id="___sec23" class="anchor">Simple example, head or tail </h2>
<p>
<h2 id="___sec23" class="anchor">Random forests </h2>
<p>
Random forests provide an improvement over bagged trees by way of a
small tweak that decorrelates the trees.
<p>
As in bagging, we build a
number of decision trees on bootstrapped training samples. But when
building these decision trees, each time a split in a tree is
considered, a random sample of \( m \) predictors is chosen as split
candidates from the full set of \( p \) predictors. The split is allowed to
use only one of those \( m \) predictors.
<p>
A fresh sample of \( m \) predictors is
taken at each split, and typically we choose
$$
m\approx \sqrt{p}.
$$
In building a random forest, at
each split in the tree, the algorithm is not even allowed to consider
a majority of the available predictors.
<p>
The reason for this is rather clever. Suppose that there is one very
strong predictor in the data set, along with a number of other
moderately strong predictors. Then in the collection of bagged
variable importance random forest trees, most or all of the trees will
use this strong predictor in the top split. Consequently, all of the
bagged trees will look quite similar to each other. Hence the
predictions from the bagged trees will be highly correlated.
Unfortunately, averaging many highly correlated quantities does not lead
to as large of a reduction in variance as averaging many uncorrelated
quanti- ties. In particular, this means that bagging will not lead to
a substantial reduction in variance over a single tree in this
setting.
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>heads_proba <span style="color: #666666">=</span> <span style="color: #666666">0.51</span>
coin_tosses <span style="color: #666666">=</span> (np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">10000</span>, <span style="color: #666666">10</span>) <span style="color: #666666">&lt;</span> heads_proba)<span style="color: #666666">.</span>astype(np<span style="color: #666666">.</span>int32)
cumulative_heads_ratio <span style="color: #666666">=</span> np<span style="color: #666666">.</span>cumsum(coin_tosses, axis<span style="color: #666666">=0</span>) <span style="color: #666666">/</span> np<span style="color: #666666">.</span>arange(<span style="color: #666666">1</span>, <span style="color: #666666">10001</span>)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">8</span>,<span style="color: #666666">3.5</span>))
plt<span style="color: #666666">.</span>plot(cumulative_heads_ratio)
plt<span style="color: #666666">.</span>plot([<span style="color: #666666">0</span>, <span style="color: #666666">10000</span>], [<span style="color: #666666">0.51</span>, <span style="color: #666666">0.51</span>], <span style="color: #BA2121">&quot;k--&quot;</span>, linewidth<span style="color: #666666">=2</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;51%&quot;</span>)
plt<span style="color: #666666">.</span>plot([<span style="color: #666666">0</span>, <span style="color: #666666">10000</span>], [<span style="color: #666666">0.5</span>, <span style="color: #666666">0.5</span>], <span style="color: #BA2121">&quot;k-&quot;</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;50%&quot;</span>)
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&quot;Number of coin tosses&quot;</span>)
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">&quot;Heads ratio&quot;</span>)
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">&quot;lower right&quot;</span>)
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>, <span style="color: #666666">10000</span>, <span style="color: #666666">0.42</span>, <span style="color: #666666">0.58</span>])
plt<span style="color: #666666">.</span>show()
</pre></div>
<p>
<p>
<!-- navigation buttons at the bottom of the page -->
@@ -197,7 +217,6 @@ plt<span style="color: #666666">.</span>show()
<li><a href="._DecisionTrees-bs027.html">28</a></li>
<li><a href="._DecisionTrees-bs028.html">29</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs025.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs025.html
+65 -92
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,45 +157,21 @@ MathJax.Hub.Config({
<a name="part0025"></a>
<!-- !split -->
<h2 id="___sec24" class="anchor">Random forests </h2>
<h2 id="___sec24" class="anchor">A simple scikit-learn example </h2>
<p>
Random forests provide an improvement over bagged trees by way of a
small tweak that decorrelates the trees.
<p>
As in bagging, we build a
number of decision trees on bootstrapped training samples. But when
building these decision trees, each time a split in a tree is
considered, a random sample of \( m \) predictors is chosen as split
candidates from the full set of \( p \) predictors. The split is allowed to
use only one of those \( m \) predictors.
<p>
A fresh sample of \( m \) predictors is
taken at each split, and typically we choose
$$
m\approx \sqrt{p}.
$$
In building a random forest, at
each split in the tree, the algorithm is not even allowed to consider
a majority of the available predictors.
<p>
The reason for this is rather clever. Suppose that there is one very
strong predictor in the data set, along with a number of other
moderately strong predictors. Then in the collection of bagged
variable importance random forest trees, most or all of the trees will
use this strong predictor in the top split. Consequently, all of the
bagged trees will look quite similar to each other. Hence the
predictions from the bagged trees will be highly correlated.
Unfortunately, averaging many highly correlated quantities does not lead
to as large of a reduction in variance as averaging many uncorrelated
quanti- ties. In particular, this means that bagging will not lead to
a substantial reduction in variance over a single tree in this
setting.
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> LabelEncoder
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
<span style="color: #408080; font-style: italic"># Data set not specificied</span>
X <span style="color: #666666">=</span> dataset<span style="color: #666666">.</span>XXX
Y <span style="color: #666666">=</span> dataset<span style="color: #666666">.</span>YYY
<span style="color: #408080; font-style: italic">#Instantiate the model with 100 trees and entropy as splitting criteria</span>
Random_Forest_model <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=100</span>,criterion<span style="color: #666666">=</span><span style="color: #BA2121">&quot;entropy&quot;</span>)
<span style="color: #408080; font-style: italic">#Cross validation</span>
accuracy <span style="color: #666666">=</span> cross_validate(Random_Forest_model,X,Y,cv<span style="color: #666666">=10</span>)[<span style="color: #BA2121">&#39;test_score&#39;</span>]
</pre></div>
<p>
<p>
<!-- navigation buttons at the bottom of the page -->
@@ -218,7 +192,6 @@ setting.
<li><a href="._DecisionTrees-bs027.html">28</a></li>
<li><a href="._DecisionTrees-bs028.html">29</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs026.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs026.html
+103 -66
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,20 +157,60 @@ MathJax.Hub.Config({
<a name="part0026"></a>
<!-- !split -->
<h2 id="___sec25" class="anchor">A simple scikit-learn example </h2>
<h2 id="___sec25" class="anchor">Please, not the moons again! </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> LabelEncoder
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
<span style="color: #408080; font-style: italic"># Data set not specificied</span>
X <span style="color: #666666">=</span> dataset<span style="color: #666666">.</span>XXX
Y <span style="color: #666666">=</span> dataset<span style="color: #666666">.</span>YYY
<span style="color: #408080; font-style: italic">#Instantiate the model with 100 trees and entropy as splitting criteria</span>
Random_Forest_model <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=100</span>,criterion<span style="color: #666666">=</span><span style="color: #BA2121">&quot;entropy&quot;</span>)
<span style="color: #408080; font-style: italic">#Cross validation</span>
accuracy <span style="color: #666666">=</span> cross_validate(Random_Forest_model,X,Y,cv<span style="color: #666666">=10</span>)[<span style="color: #BA2121">&#39;test_score&#39;</span>]
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_moons
X, y <span style="color: #666666">=</span> make_moons(n_samples<span style="color: #666666">=500</span>, noise<span style="color: #666666">=0.30</span>, random_state<span style="color: #666666">=42</span>)
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(X, y, random_state<span style="color: #666666">=42</span>)
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> VotingClassifier
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
log_clf <span style="color: #666666">=</span> LogisticRegression(random_state<span style="color: #666666">=42</span>)
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(random_state<span style="color: #666666">=42</span>)
svm_clf <span style="color: #666666">=</span> SVC(random_state<span style="color: #666666">=42</span>)
voting_clf <span style="color: #666666">=</span> VotingClassifier(
estimators<span style="color: #666666">=</span>[(<span style="color: #BA2121">&#39;lr&#39;</span>, log_clf), (<span style="color: #BA2121">&#39;rf&#39;</span>, rnd_clf), (<span style="color: #BA2121">&#39;svc&#39;</span>, svm_clf)],
voting<span style="color: #666666">=</span><span style="color: #BA2121">&#39;hard&#39;</span>)
voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
</pre></div>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
<span style="color: #008000; font-weight: bold">for</span> clf <span style="color: #AA22FF; font-weight: bold">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
clf<span style="color: #666666">.</span>fit(X_train, y_train)
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_test)
<span style="color: #008000; font-weight: bold">print</span>(clf<span style="color: #666666">.</span><span style="color: #19177C">__class__</span><span style="color: #666666">.</span><span style="color: #19177C">__name__</span>, accuracy_score(y_test, y_pred))
</pre></div>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>log_clf <span style="color: #666666">=</span> LogisticRegression(random_state<span style="color: #666666">=42</span>)
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(random_state<span style="color: #666666">=42</span>)
svm_clf <span style="color: #666666">=</span> SVC(probability<span style="color: #666666">=</span><span style="color: #008000">True</span>, random_state<span style="color: #666666">=42</span>)
voting_clf <span style="color: #666666">=</span> VotingClassifier(
estimators<span style="color: #666666">=</span>[(<span style="color: #BA2121">&#39;lr&#39;</span>, log_clf), (<span style="color: #BA2121">&#39;rf&#39;</span>, rnd_clf), (<span style="color: #BA2121">&#39;svc&#39;</span>, svm_clf)],
voting<span style="color: #666666">=</span><span style="color: #BA2121">&#39;soft&#39;</span>)
voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
</pre></div>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
<span style="color: #008000; font-weight: bold">for</span> clf <span style="color: #AA22FF; font-weight: bold">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
clf<span style="color: #666666">.</span>fit(X_train, y_train)
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_test)
<span style="color: #008000; font-weight: bold">print</span>(clf<span style="color: #666666">.</span><span style="color: #19177C">__class__</span><span style="color: #666666">.</span><span style="color: #19177C">__name__</span>, accuracy_score(y_test, y_pred))
</pre></div>
<p>
<p>
@@ -193,7 +231,6 @@ accuracy <span style="color: #666666">=</span> cross_validate(Random_Forest_mode
<li><a href="._DecisionTrees-bs027.html">28</a></li>
<li><a href="._DecisionTrees-bs028.html">29</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs027.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs027.html
+91 -91
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,60 +157,63 @@ MathJax.Hub.Config({
<a name="part0027"></a>
<!-- !split -->
<h2 id="___sec26" class="anchor">Please, not the moons again! </h2>
<h2 id="___sec26" class="anchor">Bagging examples </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_moons
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> BaggingClassifier
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
X, y <span style="color: #666666">=</span> make_moons(n_samples<span style="color: #666666">=500</span>, noise<span style="color: #666666">=0.30</span>, random_state<span style="color: #666666">=42</span>)
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(X, y, random_state<span style="color: #666666">=42</span>)
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> VotingClassifier
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
log_clf <span style="color: #666666">=</span> LogisticRegression(random_state<span style="color: #666666">=42</span>)
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(random_state<span style="color: #666666">=42</span>)
svm_clf <span style="color: #666666">=</span> SVC(random_state<span style="color: #666666">=42</span>)
voting_clf <span style="color: #666666">=</span> VotingClassifier(
estimators<span style="color: #666666">=</span>[(<span style="color: #BA2121">&#39;lr&#39;</span>, log_clf), (<span style="color: #BA2121">&#39;rf&#39;</span>, rnd_clf), (<span style="color: #BA2121">&#39;svc&#39;</span>, svm_clf)],
voting<span style="color: #666666">=</span><span style="color: #BA2121">&#39;hard&#39;</span>)
voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
bag_clf <span style="color: #666666">=</span> BaggingClassifier(
DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>), n_estimators<span style="color: #666666">=500</span>,
max_samples<span style="color: #666666">=100</span>, bootstrap<span style="color: #666666">=</span><span style="color: #008000">True</span>, n_jobs<span style="color: #666666">=-1</span>, random_state<span style="color: #666666">=42</span>)
bag_clf<span style="color: #666666">.</span>fit(X_train, y_train)
y_pred <span style="color: #666666">=</span> bag_clf<span style="color: #666666">.</span>predict(X_test)
</pre></div>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
<span style="color: #008000; font-weight: bold">for</span> clf <span style="color: #AA22FF; font-weight: bold">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
clf<span style="color: #666666">.</span>fit(X_train, y_train)
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_test)
<span style="color: #008000; font-weight: bold">print</span>(clf<span style="color: #666666">.</span><span style="color: #19177C">__class__</span><span style="color: #666666">.</span><span style="color: #19177C">__name__</span>, accuracy_score(y_test, y_pred))
<span style="color: #008000; font-weight: bold">print</span>(accuracy_score(y_test, y_pred))
</pre></div>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>log_clf <span style="color: #666666">=</span> LogisticRegression(random_state<span style="color: #666666">=42</span>)
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(random_state<span style="color: #666666">=42</span>)
svm_clf <span style="color: #666666">=</span> SVC(probability<span style="color: #666666">=</span><span style="color: #008000">True</span>, random_state<span style="color: #666666">=42</span>)
voting_clf <span style="color: #666666">=</span> VotingClassifier(
estimators<span style="color: #666666">=</span>[(<span style="color: #BA2121">&#39;lr&#39;</span>, log_clf), (<span style="color: #BA2121">&#39;rf&#39;</span>, rnd_clf), (<span style="color: #BA2121">&#39;svc&#39;</span>, svm_clf)],
voting<span style="color: #666666">=</span><span style="color: #BA2121">&#39;soft&#39;</span>)
voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>tree_clf <span style="color: #666666">=</span> DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>)
tree_clf<span style="color: #666666">.</span>fit(X_train, y_train)
y_pred_tree <span style="color: #666666">=</span> tree_clf<span style="color: #666666">.</span>predict(X_test)
<span style="color: #008000; font-weight: bold">print</span>(accuracy_score(y_test, y_pred_tree))
</pre></div>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">matplotlib.colors</span> <span style="color: #008000; font-weight: bold">import</span> ListedColormap
<span style="color: #008000; font-weight: bold">for</span> clf <span style="color: #AA22FF; font-weight: bold">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
clf<span style="color: #666666">.</span>fit(X_train, y_train)
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_test)
<span style="color: #008000; font-weight: bold">print</span>(clf<span style="color: #666666">.</span><span style="color: #19177C">__class__</span><span style="color: #666666">.</span><span style="color: #19177C">__name__</span>, accuracy_score(y_test, y_pred))
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_decision_boundary</span>(clf, X, y, axes<span style="color: #666666">=</span>[<span style="color: #666666">-1.5</span>, <span style="color: #666666">2.5</span>, <span style="color: #666666">-1</span>, <span style="color: #666666">1.5</span>], alpha<span style="color: #666666">=0.5</span>, contour<span style="color: #666666">=</span><span style="color: #008000">True</span>):
x1s <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">0</span>], axes[<span style="color: #666666">1</span>], <span style="color: #666666">100</span>)
x2s <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">2</span>], axes[<span style="color: #666666">3</span>], <span style="color: #666666">100</span>)
x1, x2 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>meshgrid(x1s, x2s)
X_new <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[x1<span style="color: #666666">.</span>ravel(), x2<span style="color: #666666">.</span>ravel()]
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_new)<span style="color: #666666">.</span>reshape(x1<span style="color: #666666">.</span>shape)
custom_cmap <span style="color: #666666">=</span> ListedColormap([<span style="color: #BA2121">&#39;#fafab0&#39;</span>,<span style="color: #BA2121">&#39;#9898ff&#39;</span>,<span style="color: #BA2121">&#39;#a0faa0&#39;</span>])
plt<span style="color: #666666">.</span>contourf(x1, x2, y_pred, alpha<span style="color: #666666">=0.3</span>, cmap<span style="color: #666666">=</span>custom_cmap)
<span style="color: #008000; font-weight: bold">if</span> contour:
custom_cmap2 <span style="color: #666666">=</span> ListedColormap([<span style="color: #BA2121">&#39;#7d7d58&#39;</span>,<span style="color: #BA2121">&#39;#4c4c7f&#39;</span>,<span style="color: #BA2121">&#39;#507d50&#39;</span>])
plt<span style="color: #666666">.</span>contour(x1, x2, y_pred, cmap<span style="color: #666666">=</span>custom_cmap2, alpha<span style="color: #666666">=0.8</span>)
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==0</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==0</span>], <span style="color: #BA2121">&quot;yo&quot;</span>, alpha<span style="color: #666666">=</span>alpha)
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==1</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==1</span>], <span style="color: #BA2121">&quot;bs&quot;</span>, alpha<span style="color: #666666">=</span>alpha)
plt<span style="color: #666666">.</span>axis(axes)
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r&quot;$x_1$&quot;</span>, fontsize<span style="color: #666666">=18</span>)
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r&quot;$x_2$&quot;</span>, fontsize<span style="color: #666666">=18</span>, rotation<span style="color: #666666">=0</span>)
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>,<span style="color: #666666">4</span>))
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
plot_decision_boundary(tree_clf, X, y)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;Decision Tree&quot;</span>, fontsize<span style="color: #666666">=14</span>)
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
plot_decision_boundary(bag_clf, X, y)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;Decision Trees with Bagging&quot;</span>, fontsize<span style="color: #666666">=14</span>)
plt<span style="color: #666666">.</span>show()
</pre></div>
<p>
<p>
@@ -232,7 +233,6 @@ voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
<li class="active"><a href="._DecisionTrees-bs027.html">28</a></li>
<li><a href="._DecisionTrees-bs028.html">29</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs028.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs028.html
+65 -107
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -159,63 +157,24 @@ MathJax.Hub.Config({
<a name="part0028"></a>
<!-- !split -->
<h2 id="___sec27" class="anchor">Bagging examples </h2>
<h2 id="___sec27" class="anchor">Then random forests </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> BaggingClassifier
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>bag_clf <span style="color: #666666">=</span> BaggingClassifier(
DecisionTreeClassifier(splitter<span style="color: #666666">=</span><span style="color: #BA2121">&quot;random&quot;</span>, max_leaf_nodes<span style="color: #666666">=16</span>, random_state<span style="color: #666666">=42</span>),
n_estimators<span style="color: #666666">=500</span>, max_samples<span style="color: #666666">=1.0</span>, bootstrap<span style="color: #666666">=</span><span style="color: #008000">True</span>, n_jobs<span style="color: #666666">=-1</span>, random_state<span style="color: #666666">=42</span>)
</pre></div>
<p>
bag_clf <span style="color: #666666">=</span> BaggingClassifier(
DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>), n_estimators<span style="color: #666666">=500</span>,
max_samples<span style="color: #666666">=100</span>, bootstrap<span style="color: #666666">=</span><span style="color: #008000">True</span>, n_jobs<span style="color: #666666">=-1</span>, random_state<span style="color: #666666">=42</span>)
bag_clf<span style="color: #666666">.</span>fit(X_train, y_train)
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>bag_clf<span style="color: #666666">.</span>fit(X_train, y_train)
y_pred <span style="color: #666666">=</span> bag_clf<span style="color: #666666">.</span>predict(X_test)
</pre></div>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
<span style="color: #008000; font-weight: bold">print</span>(accuracy_score(y_test, y_pred))
</pre></div>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>tree_clf <span style="color: #666666">=</span> DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>)
tree_clf<span style="color: #666666">.</span>fit(X_train, y_train)
y_pred_tree <span style="color: #666666">=</span> tree_clf<span style="color: #666666">.</span>predict(X_test)
<span style="color: #008000; font-weight: bold">print</span>(accuracy_score(y_test, y_pred_tree))
</pre></div>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">matplotlib.colors</span> <span style="color: #008000; font-weight: bold">import</span> ListedColormap
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_decision_boundary</span>(clf, X, y, axes<span style="color: #666666">=</span>[<span style="color: #666666">-1.5</span>, <span style="color: #666666">2.5</span>, <span style="color: #666666">-1</span>, <span style="color: #666666">1.5</span>], alpha<span style="color: #666666">=0.5</span>, contour<span style="color: #666666">=</span><span style="color: #008000">True</span>):
x1s <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">0</span>], axes[<span style="color: #666666">1</span>], <span style="color: #666666">100</span>)
x2s <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">2</span>], axes[<span style="color: #666666">3</span>], <span style="color: #666666">100</span>)
x1, x2 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>meshgrid(x1s, x2s)
X_new <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[x1<span style="color: #666666">.</span>ravel(), x2<span style="color: #666666">.</span>ravel()]
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_new)<span style="color: #666666">.</span>reshape(x1<span style="color: #666666">.</span>shape)
custom_cmap <span style="color: #666666">=</span> ListedColormap([<span style="color: #BA2121">&#39;#fafab0&#39;</span>,<span style="color: #BA2121">&#39;#9898ff&#39;</span>,<span style="color: #BA2121">&#39;#a0faa0&#39;</span>])
plt<span style="color: #666666">.</span>contourf(x1, x2, y_pred, alpha<span style="color: #666666">=0.3</span>, cmap<span style="color: #666666">=</span>custom_cmap)
<span style="color: #008000; font-weight: bold">if</span> contour:
custom_cmap2 <span style="color: #666666">=</span> ListedColormap([<span style="color: #BA2121">&#39;#7d7d58&#39;</span>,<span style="color: #BA2121">&#39;#4c4c7f&#39;</span>,<span style="color: #BA2121">&#39;#507d50&#39;</span>])
plt<span style="color: #666666">.</span>contour(x1, x2, y_pred, cmap<span style="color: #666666">=</span>custom_cmap2, alpha<span style="color: #666666">=0.8</span>)
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==0</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==0</span>], <span style="color: #BA2121">&quot;yo&quot;</span>, alpha<span style="color: #666666">=</span>alpha)
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==1</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==1</span>], <span style="color: #BA2121">&quot;bs&quot;</span>, alpha<span style="color: #666666">=</span>alpha)
plt<span style="color: #666666">.</span>axis(axes)
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r&quot;$x_1$&quot;</span>, fontsize<span style="color: #666666">=18</span>)
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r&quot;$x_2$&quot;</span>, fontsize<span style="color: #666666">=18</span>, rotation<span style="color: #666666">=0</span>)
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>,<span style="color: #666666">4</span>))
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
plot_decision_boundary(tree_clf, X, y)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;Decision Tree&quot;</span>, fontsize<span style="color: #666666">=14</span>)
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
plot_decision_boundary(bag_clf, X, y)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;Decision Trees with Bagging&quot;</span>, fontsize<span style="color: #666666">=14</span>)
plt<span style="color: #666666">.</span>show()
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=500</span>, max_leaf_nodes<span style="color: #666666">=16</span>, n_jobs<span style="color: #666666">=-1</span>, random_state<span style="color: #666666">=42</span>)
rnd_clf<span style="color: #666666">.</span>fit(X_train, y_train)
y_pred_rf <span style="color: #666666">=</span> rnd_clf<span style="color: #666666">.</span>predict(X_test)
np<span style="color: #666666">.</span>sum(y_pred <span style="color: #666666">==</span> y_pred_rf) <span style="color: #666666">/</span> <span style="color: #008000">len</span>(y_pred)
</pre></div>
<p>
<p>
@@ -234,7 +193,6 @@ plt<span style="color: #666666">.</span>show()
<li><a href="._DecisionTrees-bs027.html">28</a></li>
<li class="active"><a href="._DecisionTrees-bs028.html">29</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/._DecisionTrees-bs029.html
+56 -75
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -157,28 +155,13 @@ MathJax.Hub.Config({
<p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
<a name="part0029"></a>
<!-- !split -->
<!-- !split -->
<h2 id="___sec28" class="anchor">Boosting and more </h2>
More material to come here.
<h2 id="___sec28" class="anchor">Then random forests </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>bag_clf <span style="color: #666666">=</span> BaggingClassifier(
DecisionTreeClassifier(splitter<span style="color: #666666">=</span><span style="color: #BA2121">&quot;random&quot;</span>, max_leaf_nodes<span style="color: #666666">=16</span>, random_state<span style="color: #666666">=42</span>),
n_estimators<span style="color: #666666">=500</span>, max_samples<span style="color: #666666">=1.0</span>, bootstrap<span style="color: #666666">=</span><span style="color: #008000">True</span>, n_jobs<span style="color: #666666">=-1</span>, random_state<span style="color: #666666">=42</span>)
</pre></div>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>bag_clf<span style="color: #666666">.</span>fit(X_train, y_train)
y_pred <span style="color: #666666">=</span> bag_clf<span style="color: #666666">.</span>predict(X_test)
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=500</span>, max_leaf_nodes<span style="color: #666666">=16</span>, n_jobs<span style="color: #666666">=-1</span>, random_state<span style="color: #666666">=42</span>)
rnd_clf<span style="color: #666666">.</span>fit(X_train, y_train)
y_pred_rf <span style="color: #666666">=</span> rnd_clf<span style="color: #666666">.</span>predict(X_test)
np<span style="color: #666666">.</span>sum(y_pred <span style="color: #666666">==</span> y_pred_rf) <span style="color: #666666">/</span> <span style="color: #008000">len</span>(y_pred)
</pre></div>
<p>
<p>
<!-- navigation buttons at the bottom of the page -->
<ul class="pagination">
@@ -194,8 +177,6 @@ np<span style="color: #666666">.</span>sum(y_pred <span style="color: #666666">=
<li><a href="._DecisionTrees-bs027.html">28</a></li>
<li><a href="._DecisionTrees-bs028.html">29</a></li>
<li class="active"><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs030.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DecisionTrees/html/DecisionTrees-bs.html
+55 -57
View File
@@ -44,39 +44,38 @@ Automatically generated HTML file from DocOnce source
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -117,33 +116,32 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Maxwell-Boltzmann velocity distribution</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Boosting and more</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">A schematic procedure</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">A classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Writing your own code for a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Back to moons again</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Classification again: The zoo data</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Simple example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">A simple scikit-learn example</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Please, not the moons again!</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Bagging examples</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Then random forests</a></li>
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Boosting and more</a></li>
</ul>
</li>
@@ -178,7 +176,7 @@ MathJax.Hub.Config({
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
<br>
<p>
<center><h4>Dec 17, 2018</h4></center> <!-- date -->
<center><h4>Oct 24, 2019</h4></center> <!-- date -->
<br>
<p>
@@ -202,7 +200,7 @@ MathJax.Hub.Config({
<li><a href="._DecisionTrees-bs008.html">9</a></li>
<li><a href="._DecisionTrees-bs009.html">10</a></li>
<li><a href="">...</a></li>
<li><a href="._DecisionTrees-bs030.html">31</a></li>
<li><a href="._DecisionTrees-bs029.html">30</a></li>
<li><a href="._DecisionTrees-bs001.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
@@ -220,7 +218,7 @@ MathJax.Hub.Config({
<center style="font-size:80%">
<!-- copyright --> &copy; 1999-2018, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
<!-- copyright --> &copy; 1999-2019, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
</center>
doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
+31 -130
View File
@@ -148,12 +148,12 @@ MathJax.Hub.Config({
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
<br>
<p>&nbsp;<br>
<center><h4>Dec 17, 2018</h4></center> <!-- date -->
<center><h4>Oct 24, 2019</h4></center> <!-- date -->
<br>
<p>
<center style="font-size:80%">
<!-- copyright --> &copy; 1999-2018, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
<!-- copyright --> &copy; 1999-2019, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
</center>
</section>
@@ -309,109 +309,7 @@ plt.show()
<section>
<h2 id="___sec3">Maxwell-Boltzmann velocity distribution </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #228B22"># Program to test the Metropolis algorithm with one particle at given temp in</span>
<span style="color: #228B22"># one dimension</span>
<span style="color: #228B22">#!/usr/bin/env python</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.mlab</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">mlab</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">random</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">math</span> <span style="color: #8B008B; font-weight: bold">import</span> sqrt, exp, log
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> PolynomialFeatures
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearRegression
<span style="color: #228B22"># initialize the rng with a seed</span>
random.seed()
<span style="color: #228B22"># Hard coding of input parameters</span>
MCcycles = <span style="color: #B452CD">100000</span>
Temperature = <span style="color: #B452CD">2.0</span>
beta = <span style="color: #B452CD">1.</span>/Temperature
InitialVelocity = -<span style="color: #B452CD">2.0</span>
CurrentVelocity = InitialVelocity
Energy = <span style="color: #B452CD">0.5</span>*InitialVelocity*InitialVelocity
VelocityRange = <span style="color: #B452CD">10</span>*sqrt(Temperature)
VelocityStep = <span style="color: #B452CD">2</span>*VelocityRange/<span style="color: #B452CD">10.</span>
AverageEnergy = Energy
AverageEnergy2 = Energy*Energy
VelocityValues = np.zeros(MCcycles)
<span style="color: #228B22"># The Monte Carlo sampling with Metropolis starts here</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span> (<span style="color: #B452CD">1</span>, MCcycles, <span style="color: #B452CD">1</span>):
TrialVelocity = CurrentVelocity + (<span style="color: #B452CD">2.0</span>*random.random() - <span style="color: #B452CD">1.0</span>)*VelocityStep
EnergyChange = <span style="color: #B452CD">0.5</span>*(TrialVelocity*TrialVelocity -CurrentVelocity*CurrentVelocity);
<span style="color: #8B008B; font-weight: bold">if</span> random.random() &lt;= exp(-beta*EnergyChange):
CurrentVelocity = TrialVelocity
Energy += EnergyChange
VelocityValues[i] = CurrentVelocity
AverageEnergy += Energy
AverageEnergy2 += Energy*Energy
<span style="color: #228B22">#Final averages</span>
AverageEnergy = AverageEnergy/MCcycles
AverageEnergy2 = AverageEnergy2/MCcycles
Variance = AverageEnergy2 - AverageEnergy*AverageEnergy
<span style="color: #8B008B; font-weight: bold">print</span>(AverageEnergy, Variance)
n, bins, patches = plt.hist(VelocityValues, <span style="color: #B452CD">400</span>, facecolor=<span style="color: #CD5555">&#39;green&#39;</span>)
plt.xlabel(<span style="color: #CD5555">&#39;$v$&#39;</span>)
plt.ylabel(<span style="color: #CD5555">&#39;Velocity distribution P(v)&#39;</span>)
plt.title(<span style="color: #CD5555">r&#39;Velocity histogram at $k_BT=2$&#39;</span>)
plt.axis([-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">600</span>])
plt.grid(<span style="color: #658b00">True</span>)
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">collections</span> <span style="color: #8B008B; font-weight: bold">import</span> Counter
<span style="color: #228B22">#print (Counter(VelocityValues))</span>
<span style="color: #8B008B; font-weight: bold">print</span> (VelocityValues[:<span style="color: #B452CD">20</span>])
VelocityValues=<span style="color: #658b00">list</span>(Counter(VelocityValues).keys())
d=<span style="color: #658b00">list</span>(Counter(VelocityValues).values())
VelocityValues=np.asarray(VelocityValues)[:, np.newaxis]
d=np.asarray(d)
<span style="color: #8B008B; font-weight: bold">print</span> (VelocityValues.shape, d.shape)
plt.scatter(VelocityValues, d)
plt.show()
<span style="color: #228B22">#2nd Degree Polynomial</span>
poly_feat=PolynomialFeatures(degree=<span style="color: #B452CD">20</span>, include_bias=<span style="color: #658b00">False</span>)
X_poly=poly_feat.fit_transform(VelocityValues)
lin_reg=LinearRegression()
poly_fit=lin_reg.fit(X_poly,d)
y_plot=poly_fit.predict(X_poly)
plt.title(<span style="color: #CD5555">&quot;Polynomial Fit&quot;</span>)
plt.plot(VelocityValues, y_plot, color=<span style="color: #CD5555">&#39;black&#39;</span>, label=<span style="color: #CD5555">&quot;Fit&quot;</span>)
plt.show()
<span style="color: #228B22">#Decision Trees</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> DecisionTreeRegressor
regr_1=DecisionTreeRegressor(max_depth=<span style="color: #B452CD">2</span>)
regr_2=DecisionTreeRegressor(max_depth=<span style="color: #B452CD">5</span>)
regr_3=DecisionTreeRegressor(max_depth=<span style="color: #B452CD">7</span>)
regr_1.fit(VelocityValues, d)
regr_2.fit(VelocityValues, d)
regr_3.fit(VelocityValues, d)
X_test = np.arange(<span style="color: #B452CD">0.0</span>, MCcycles, <span style="color: #B452CD">0.01</span>)[:, np.newaxis]
y_1=regr_1.predict(X_test)
y_2=regr_2.predict(X_test)
y_3=regr_3.predict(X_test)
plt.title(<span style="color: #CD5555">&quot;Decision Tree&quot;</span>)
plt.plot(X_test, y_1, color=<span style="color: #CD5555">&quot;red&quot;</span>, label=<span style="color: #CD5555">&quot;max_depth=2&quot;</span>, linewidth=<span style="color: #B452CD">2</span>)
plt.plot(X_test, y_2, color=<span style="color: #CD5555">&quot;green&quot;</span>, label=<span style="color: #CD5555">&quot;max_depth=5&quot;</span>, linewidth=<span style="color: #B452CD">2</span>)
plt.plot(X_test, y_3, color=<span style="color: #CD5555">&quot;m&quot;</span>, label=<span style="color: #CD5555">&quot;max_depth=7&quot;</span>, linewidth=<span style="color: #B452CD">2</span>)
plt.show()
</pre></div>
</section>
<section>
<h2 id="___sec4">Building a tree, regression </h2>
<h2 id="___sec3">Building a tree, regression </h2>
<p>
There are mainly two steps
@@ -447,7 +345,7 @@ box.
<section>
<h2 id="___sec5">A top-down approach, recursive binary splitting </h2>
<h2 id="___sec4">A top-down approach, recursive binary splitting </h2>
<p>
Unfortunately, it is computationally infeasible to consider every
@@ -466,7 +364,7 @@ better tree in some future step.
<section>
<h2 id="___sec6">Making a tree </h2>
<h2 id="___sec5">Making a tree </h2>
<p>
In order to implement the recursive binary splitting we start by selecting
@@ -519,7 +417,7 @@ region contains more than five observations.
<section>
<h2 id="___sec7">Pruning the tree </h2>
<h2 id="___sec6">Pruning the tree </h2>
<p>
The above procedure is rather straightforward, but leads often to
@@ -538,7 +436,7 @@ parameter \( \alpha \).
<section>
<h2 id="___sec8">Cost complexity pruning </h2>
<h2 id="___sec7">Cost complexity pruning </h2>
For each value of \( \alpha \) there corresponds a subtree \( T \in T_0 \) such that
<p>&nbsp;<br>
$$
@@ -571,7 +469,7 @@ subtree corresponding to \( \alpha \).
<section>
<h2 id="___sec9">A schematic procedure </h2>
<h2 id="___sec8">A schematic procedure </h2>
<p>
<div class="alert alert-block alert-block alert-text-normal">
@@ -596,7 +494,7 @@ subtree corresponding to \( \alpha \).
<section>
<h2 id="___sec10">A classification tree </h2>
<h2 id="___sec9">A classification tree </h2>
<p>
A classification tree is very similar to a regression tree, except
@@ -615,7 +513,7 @@ fall into that region.
<section>
<h2 id="___sec11">Growing a classification tree </h2>
<h2 id="___sec10">Growing a classification tree </h2>
<p>
The task of growing a
@@ -639,7 +537,7 @@ than is the classification error rate.
<section>
<h2 id="___sec12">Classification tree, how to split nodes </h2>
<h2 id="___sec11">Classification tree, how to split nodes </h2>
If our targets are the outcome of a classification process that takes for example
\( k=1,2,\dots,K \) values, the only thing we need to think of is to set up the splitting criteria for each node.
@@ -686,12 +584,15 @@ $$
<section>
<h2 id="___sec13">Entropy and the ID3 algorithm </h2>
<h2 id="___sec12">Entropy and the ID3 algorithm </h2>
<p>
More text to come here.
</section>
<section>
<h2 id="___sec14">Writing your own code for a classification tree </h2>
<h2 id="___sec13">Writing your own code for a classification tree </h2>
<p>
@@ -784,7 +685,7 @@ $$
<section>
<h2 id="___sec15">Back to moons again </h2>
<h2 id="___sec14">Back to moons again </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -857,7 +758,7 @@ plt.show()
<section>
<h2 id="___sec16">Playing around with regions </h2>
<h2 id="___sec15">Playing around with regions </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -886,7 +787,7 @@ plt.show()
<section>
<h2 id="___sec17">Regression trees </h2>
<h2 id="___sec16">Regression trees </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -909,7 +810,7 @@ tree_reg.fit(X, y)
<section>
<h2 id="___sec18">Final regressor code </h2>
<h2 id="___sec17">Final regressor code </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -988,7 +889,7 @@ plt.show()
<section>
<h2 id="___sec19">Classification again: The zoo data </h2>
<h2 id="___sec18">Classification again: The zoo data </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -1017,7 +918,7 @@ prediction = tree.predict(test_features)
<section>
<h2 id="___sec20">Pros and cons of trees, pros </h2>
<h2 id="___sec19">Pros and cons of trees, pros </h2>
<ul>
<p><li> White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines)</li>
@@ -1032,7 +933,7 @@ prediction = tree.predict(test_features)
<section>
<h2 id="___sec21">Disadvantages </h2>
<h2 id="___sec20">Disadvantages </h2>
<ul>
<p><li> Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches</li>
@@ -1050,7 +951,7 @@ However, by aggregating many decision trees, using methods like bagging, random
<section>
<h2 id="___sec22">Bagging </h2>
<h2 id="___sec21">Bagging </h2>
<p>
The <b>plain</b> decision trees suffer from high
@@ -1093,7 +994,7 @@ predictor, averaged over all \( B \) trees.
<section>
<h2 id="___sec23">Simple example, head or tail </h2>
<h2 id="___sec22">Simple example, head or tail </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -1114,7 +1015,7 @@ plt.show()
<section>
<h2 id="___sec24">Random forests </h2>
<h2 id="___sec23">Random forests </h2>
<p>
Random forests provide an improvement over bagged trees by way of a
@@ -1158,7 +1059,7 @@ setting.
<section>
<h2 id="___sec25">A simple scikit-learn example </h2>
<h2 id="___sec24">A simple scikit-learn example </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -1177,7 +1078,7 @@ accuracy = cross_validate(Random_Forest_model,X,Y,cv=<span style="color: #B452CD
<section>
<h2 id="___sec26">Please, not the moons again! </h2>
<h2 id="___sec25">Please, not the moons again! </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -1236,7 +1137,7 @@ voting_clf.fit(X_train, y_train)
<section>
<h2 id="___sec27">Bagging examples </h2>
<h2 id="___sec26">Bagging examples </h2>
<p>
@@ -1298,7 +1199,7 @@ plt.show()
<section>
<h2 id="___sec28">Then random forests </h2>
<h2 id="___sec27">Then random forests </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -1321,7 +1222,7 @@ np.sum(y_pred == y_pred_rf) / <span style="color: #658b00">len</span>(y_pred)
<section>
<h2 id="___sec29">Boosting and more </h2>
<h2 id="___sec28">Boosting and more </h2>
More material to come here.
</section>
doc/pub/DecisionTrees/html/DecisionTrees-solarized.html
+57 -156
View File
@@ -64,39 +64,38 @@ div { text-align: justify; text-justify: inter-word; }
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -138,7 +137,7 @@ MathJax.Hub.Config({
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
<br>
<p>
<center><h4>Dec 17, 2018</h4></center> <!-- date -->
<center><h4>Oct 24, 2019</h4></center> <!-- date -->
<br>
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
@@ -291,108 +290,7 @@ plt.show()
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec3">Maxwell-Boltzmann velocity distribution </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #228B22"># Program to test the Metropolis algorithm with one particle at given temp in</span>
<span style="color: #228B22"># one dimension</span>
<span style="color: #228B22">#!/usr/bin/env python</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.mlab</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">mlab</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">random</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">math</span> <span style="color: #8B008B; font-weight: bold">import</span> sqrt, exp, log
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> PolynomialFeatures
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearRegression
<span style="color: #228B22"># initialize the rng with a seed</span>
random.seed()
<span style="color: #228B22"># Hard coding of input parameters</span>
MCcycles = <span style="color: #B452CD">100000</span>
Temperature = <span style="color: #B452CD">2.0</span>
beta = <span style="color: #B452CD">1.</span>/Temperature
InitialVelocity = -<span style="color: #B452CD">2.0</span>
CurrentVelocity = InitialVelocity
Energy = <span style="color: #B452CD">0.5</span>*InitialVelocity*InitialVelocity
VelocityRange = <span style="color: #B452CD">10</span>*sqrt(Temperature)
VelocityStep = <span style="color: #B452CD">2</span>*VelocityRange/<span style="color: #B452CD">10.</span>
AverageEnergy = Energy
AverageEnergy2 = Energy*Energy
VelocityValues = np.zeros(MCcycles)
<span style="color: #228B22"># The Monte Carlo sampling with Metropolis starts here</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span> (<span style="color: #B452CD">1</span>, MCcycles, <span style="color: #B452CD">1</span>):
TrialVelocity = CurrentVelocity + (<span style="color: #B452CD">2.0</span>*random.random() - <span style="color: #B452CD">1.0</span>)*VelocityStep
EnergyChange = <span style="color: #B452CD">0.5</span>*(TrialVelocity*TrialVelocity -CurrentVelocity*CurrentVelocity);
<span style="color: #8B008B; font-weight: bold">if</span> random.random() &lt;= exp(-beta*EnergyChange):
CurrentVelocity = TrialVelocity
Energy += EnergyChange
VelocityValues[i] = CurrentVelocity
AverageEnergy += Energy
AverageEnergy2 += Energy*Energy
<span style="color: #228B22">#Final averages</span>
AverageEnergy = AverageEnergy/MCcycles
AverageEnergy2 = AverageEnergy2/MCcycles
Variance = AverageEnergy2 - AverageEnergy*AverageEnergy
<span style="color: #8B008B; font-weight: bold">print</span>(AverageEnergy, Variance)
n, bins, patches = plt.hist(VelocityValues, <span style="color: #B452CD">400</span>, facecolor=<span style="color: #CD5555">&#39;green&#39;</span>)
plt.xlabel(<span style="color: #CD5555">&#39;$v$&#39;</span>)
plt.ylabel(<span style="color: #CD5555">&#39;Velocity distribution P(v)&#39;</span>)
plt.title(<span style="color: #CD5555">r&#39;Velocity histogram at $k_BT=2$&#39;</span>)
plt.axis([-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">600</span>])
plt.grid(<span style="color: #658b00">True</span>)
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">collections</span> <span style="color: #8B008B; font-weight: bold">import</span> Counter
<span style="color: #228B22">#print (Counter(VelocityValues))</span>
<span style="color: #8B008B; font-weight: bold">print</span> (VelocityValues[:<span style="color: #B452CD">20</span>])
VelocityValues=<span style="color: #658b00">list</span>(Counter(VelocityValues).keys())
d=<span style="color: #658b00">list</span>(Counter(VelocityValues).values())
VelocityValues=np.asarray(VelocityValues)[:, np.newaxis]
d=np.asarray(d)
<span style="color: #8B008B; font-weight: bold">print</span> (VelocityValues.shape, d.shape)
plt.scatter(VelocityValues, d)
plt.show()
<span style="color: #228B22">#2nd Degree Polynomial</span>
poly_feat=PolynomialFeatures(degree=<span style="color: #B452CD">20</span>, include_bias=<span style="color: #658b00">False</span>)
X_poly=poly_feat.fit_transform(VelocityValues)
lin_reg=LinearRegression()
poly_fit=lin_reg.fit(X_poly,d)
y_plot=poly_fit.predict(X_poly)
plt.title(<span style="color: #CD5555">&quot;Polynomial Fit&quot;</span>)
plt.plot(VelocityValues, y_plot, color=<span style="color: #CD5555">&#39;black&#39;</span>, label=<span style="color: #CD5555">&quot;Fit&quot;</span>)
plt.show()
<span style="color: #228B22">#Decision Trees</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> DecisionTreeRegressor
regr_1=DecisionTreeRegressor(max_depth=<span style="color: #B452CD">2</span>)
regr_2=DecisionTreeRegressor(max_depth=<span style="color: #B452CD">5</span>)
regr_3=DecisionTreeRegressor(max_depth=<span style="color: #B452CD">7</span>)
regr_1.fit(VelocityValues, d)
regr_2.fit(VelocityValues, d)
regr_3.fit(VelocityValues, d)
X_test = np.arange(<span style="color: #B452CD">0.0</span>, MCcycles, <span style="color: #B452CD">0.01</span>)[:, np.newaxis]
y_1=regr_1.predict(X_test)
y_2=regr_2.predict(X_test)
y_3=regr_3.predict(X_test)
plt.title(<span style="color: #CD5555">&quot;Decision Tree&quot;</span>)
plt.plot(X_test, y_1, color=<span style="color: #CD5555">&quot;red&quot;</span>, label=<span style="color: #CD5555">&quot;max_depth=2&quot;</span>, linewidth=<span style="color: #B452CD">2</span>)
plt.plot(X_test, y_2, color=<span style="color: #CD5555">&quot;green&quot;</span>, label=<span style="color: #CD5555">&quot;max_depth=5&quot;</span>, linewidth=<span style="color: #B452CD">2</span>)
plt.plot(X_test, y_3, color=<span style="color: #CD5555">&quot;m&quot;</span>, label=<span style="color: #CD5555">&quot;max_depth=7&quot;</span>, linewidth=<span style="color: #B452CD">2</span>)
plt.show()
</pre></div>
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec4">Building a tree, regression </h2>
<h2 id="___sec3">Building a tree, regression </h2>
<p>
There are mainly two steps
@@ -424,7 +322,7 @@ box.
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec5">A top-down approach, recursive binary splitting </h2>
<h2 id="___sec4">A top-down approach, recursive binary splitting </h2>
<p>
Unfortunately, it is computationally infeasible to consider every
@@ -443,7 +341,7 @@ better tree in some future step.
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec6">Making a tree </h2>
<h2 id="___sec5">Making a tree </h2>
<p>
In order to implement the recursive binary splitting we start by selecting
@@ -490,7 +388,7 @@ region contains more than five observations.
<p>
<!-- !split -->
<h2 id="___sec7">Pruning the tree </h2>
<h2 id="___sec6">Pruning the tree </h2>
<p>
The above procedure is rather straightforward, but leads often to
@@ -509,7 +407,7 @@ parameter \( \alpha \).
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec8">Cost complexity pruning </h2>
<h2 id="___sec7">Cost complexity pruning </h2>
For each value of \( \alpha \) there corresponds a subtree \( T \in T_0 \) such that
$$
\sum_{m=1}^{\overline{T}}\sum_{i:x_i\in R_m}(y_i-\overline{y}_{R_m})^2+\alpha\overline{T},
@@ -540,7 +438,7 @@ subtree corresponding to \( \alpha \).
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec9">A schematic procedure </h2>
<h2 id="___sec8">A schematic procedure </h2>
<p>
<div class="alert alert-block alert-block alert-text-normal">
@@ -566,7 +464,7 @@ subtree corresponding to \( \alpha \).
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec10">A classification tree </h2>
<h2 id="___sec9">A classification tree </h2>
<p>
A classification tree is very similar to a regression tree, except
@@ -585,7 +483,7 @@ fall into that region.
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec11">Growing a classification tree </h2>
<h2 id="___sec10">Growing a classification tree </h2>
<p>
The task of growing a
@@ -609,7 +507,7 @@ than is the classification error rate.
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec12">Classification tree, how to split nodes </h2>
<h2 id="___sec11">Classification tree, how to split nodes </h2>
If our targets are the outcome of a classification process that takes for example
\( k=1,2,\dots,K \) values, the only thing we need to think of is to set up the splitting criteria for each node.
@@ -651,12 +549,15 @@ $$
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec13">Entropy and the ID3 algorithm </h2>
<h2 id="___sec12">Entropy and the ID3 algorithm </h2>
<p>
More text to come here.
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec14">Writing your own code for a classification tree </h2>
<h2 id="___sec13">Writing your own code for a classification tree </h2>
<p>
@@ -748,7 +649,7 @@ $$
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec15">Back to moons again </h2>
<h2 id="___sec14">Back to moons again </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -820,7 +721,7 @@ plt.show()
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec16">Playing around with regions </h2>
<h2 id="___sec15">Playing around with regions </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -848,7 +749,7 @@ plt.show()
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec17">Regression trees </h2>
<h2 id="___sec16">Regression trees </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -870,7 +771,7 @@ tree_reg.fit(X, y)
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec18">Final regressor code </h2>
<h2 id="___sec17">Final regressor code </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -948,7 +849,7 @@ plt.show()
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec19">Classification again: The zoo data </h2>
<h2 id="___sec18">Classification again: The zoo data </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -976,7 +877,7 @@ prediction = tree.predict(test_features)
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec20">Pros and cons of trees, pros </h2>
<h2 id="___sec19">Pros and cons of trees, pros </h2>
<ul>
<li> White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines)</li>
@@ -990,7 +891,7 @@ prediction = tree.predict(test_features)
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec21">Disadvantages </h2>
<h2 id="___sec20">Disadvantages </h2>
<ul>
<li> Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches</li>
@@ -1007,7 +908,7 @@ However, by aggregating many decision trees, using methods like bagging, random
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec22">Bagging </h2>
<h2 id="___sec21">Bagging </h2>
<p>
The <b>plain</b> decision trees suffer from high
@@ -1050,7 +951,7 @@ predictor, averaged over all \( B \) trees.
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec23">Simple example, head or tail </h2>
<h2 id="___sec22">Simple example, head or tail </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -1070,7 +971,7 @@ plt.show()
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec24">Random forests </h2>
<h2 id="___sec23">Random forests </h2>
<p>
Random forests provide an improvement over bagged trees by way of a
@@ -1112,7 +1013,7 @@ setting.
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec25">A simple scikit-learn example </h2>
<h2 id="___sec24">A simple scikit-learn example </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -1130,7 +1031,7 @@ accuracy = cross_validate(Random_Forest_model,X,Y,cv=<span style="color: #B452CD
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec26">Please, not the moons again! </h2>
<h2 id="___sec25">Please, not the moons again! </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -1188,7 +1089,7 @@ voting_clf.fit(X_train, y_train)
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec27">Bagging examples </h2>
<h2 id="___sec26">Bagging examples </h2>
<p>
@@ -1249,7 +1150,7 @@ plt.show()
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec28">Then random forests </h2>
<h2 id="___sec27">Then random forests </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
@@ -1271,7 +1172,7 @@ np.sum(y_pred == y_pred_rf) / <span style="color: #658b00">len</span>(y_pred)
<p>
<!-- !split -->
<h2 id="___sec29">Boosting and more </h2>
<h2 id="___sec28">Boosting and more </h2>
More material to come here.
<p>
@@ -1280,7 +1181,7 @@ More material to come here.
<center style="font-size:80%">
<!-- copyright --> &copy; 1999-2018, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
<!-- copyright --> &copy; 1999-2019, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
</center>
doc/pub/DecisionTrees/html/DecisionTrees.html
+57 -156
View File
@@ -69,39 +69,38 @@ div { text-align: justify; text-justify: inter-word; }
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
('How do we set it up?', 2, None, '___sec1'),
('Decision trees and Regression', 2, None, '___sec2'),
('Maxwell-Boltzmann velocity distribution', 2, None, '___sec3'),
('Building a tree, regression', 2, None, '___sec4'),
('Building a tree, regression', 2, None, '___sec3'),
('A top-down approach, recursive binary splitting',
2,
None,
'___sec5'),
('Making a tree', 2, None, '___sec6'),
('Pruning the tree', 2, None, '___sec7'),
('Cost complexity pruning', 2, None, '___sec8'),
('A schematic procedure', 2, None, '___sec9'),
('A classification tree', 2, None, '___sec10'),
('Growing a classification tree', 2, None, '___sec11'),
('Classification tree, how to split nodes', 2, None, '___sec12'),
('Entropy and the ID3 algorithm', 2, None, '___sec13'),
'___sec4'),
('Making a tree', 2, None, '___sec5'),
('Pruning the tree', 2, None, '___sec6'),
('Cost complexity pruning', 2, None, '___sec7'),
('A schematic procedure', 2, None, '___sec8'),
('A classification tree', 2, None, '___sec9'),
('Growing a classification tree', 2, None, '___sec10'),
('Classification tree, how to split nodes', 2, None, '___sec11'),
('Entropy and the ID3 algorithm', 2, None, '___sec12'),
('Writing your own code for a classification tree',
2,
None,
'___sec14'),
('Back to moons again', 2, None, '___sec15'),
('Playing around with regions', 2, None, '___sec16'),
('Regression trees', 2, None, '___sec17'),
('Final regressor code', 2, None, '___sec18'),
('Classification again: The zoo data', 2, None, '___sec19'),
('Pros and cons of trees, pros', 2, None, '___sec20'),
('Disadvantages', 2, None, '___sec21'),
('Bagging', 2, None, '___sec22'),
('Simple example, head or tail', 2, None, '___sec23'),
('Random forests', 2, None, '___sec24'),
('A simple scikit-learn example', 2, None, '___sec25'),
('Please, not the moons again!', 2, None, '___sec26'),
('Bagging examples', 2, None, '___sec27'),
('Then random forests', 2, None, '___sec28'),
('Boosting and more', 2, None, '___sec29')]}
'___sec13'),
('Back to moons again', 2, None, '___sec14'),
('Playing around with regions', 2, None, '___sec15'),
('Regression trees', 2, None, '___sec16'),
('Final regressor code', 2, None, '___sec17'),
('Classification again: The zoo data', 2, None, '___sec18'),
('Pros and cons of trees, pros', 2, None, '___sec19'),
('Disadvantages', 2, None, '___sec20'),
('Bagging', 2, None, '___sec21'),
('Simple example, head or tail', 2, None, '___sec22'),
('Random forests', 2, None, '___sec23'),
('A simple scikit-learn example', 2, None, '___sec24'),
('Please, not the moons again!', 2, None, '___sec25'),
('Bagging examples', 2, None, '___sec26'),
('Then random forests', 2, None, '___sec27'),
('Boosting and more', 2, None, '___sec28')]}
end of tocinfo -->
<body>
@@ -143,7 +142,7 @@ MathJax.Hub.Config({
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
<br>
<p>
<center><h4>Dec 17, 2018</h4></center> <!-- date -->
<center><h4>Oct 24, 2019</h4></center> <!-- date -->
<br>
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
@@ -296,108 +295,7 @@ plt<span style="color: #666666">.</span>show()
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec3">Maxwell-Boltzmann velocity distribution </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Program to test the Metropolis algorithm with one particle at given temp in</span>
<span style="color: #408080; font-style: italic"># one dimension</span>
<span style="color: #408080; font-style: italic">#!/usr/bin/env python</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.mlab</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">mlab</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">random</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">math</span> <span style="color: #008000; font-weight: bold">import</span> sqrt, exp, log
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> PolynomialFeatures
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LinearRegression
<span style="color: #408080; font-style: italic"># initialize the rng with a seed</span>
random<span style="color: #666666">.</span>seed()
<span style="color: #408080; font-style: italic"># Hard coding of input parameters</span>
MCcycles <span style="color: #666666">=</span> <span style="color: #666666">100000</span>
Temperature <span style="color: #666666">=</span> <span style="color: #666666">2.0</span>
beta <span style="color: #666666">=</span> <span style="color: #666666">1./</span>Temperature
InitialVelocity <span style="color: #666666">=</span> <span style="color: #666666">-2.0</span>
CurrentVelocity <span style="color: #666666">=</span> InitialVelocity
Energy <span style="color: #666666">=</span> <span style="color: #666666">0.5*</span>InitialVelocity<span style="color: #666666">*</span>InitialVelocity
VelocityRange <span style="color: #666666">=</span> <span style="color: #666666">10*</span>sqrt(Temperature)
VelocityStep <span style="color: #666666">=</span> <span style="color: #666666">2*</span>VelocityRange<span style="color: #666666">/10.</span>
AverageEnergy <span style="color: #666666">=</span> Energy
AverageEnergy2 <span style="color: #666666">=</span> Energy<span style="color: #666666">*</span>Energy
VelocityValues <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(MCcycles)
<span style="color: #408080; font-style: italic"># The Monte Carlo sampling with Metropolis starts here</span>
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span> (<span style="color: #666666">1</span>, MCcycles, <span style="color: #666666">1</span>):
TrialVelocity <span style="color: #666666">=</span> CurrentVelocity <span style="color: #666666">+</span> (<span style="color: #666666">2.0*</span>random<span style="color: #666666">.</span>random() <span style="color: #666666">-</span> <span style="color: #666666">1.0</span>)<span style="color: #666666">*</span>VelocityStep
EnergyChange <span style="color: #666666">=</span> <span style="color: #666666">0.5*</span>(TrialVelocity<span style="color: #666666">*</span>TrialVelocity <span style="color: #666666">-</span>CurrentVelocity<span style="color: #666666">*</span>CurrentVelocity);
<span style="color: #008000; font-weight: bold">if</span> random<span style="color: #666666">.</span>random() <span style="color: #666666">&lt;=</span> exp(<span style="color: #666666">-</span>beta<span style="color: #666666">*</span>EnergyChange):
CurrentVelocity <span style="color: #666666">=</span> TrialVelocity
Energy <span style="color: #666666">+=</span> EnergyChange
VelocityValues[i] <span style="color: #666666">=</span> CurrentVelocity
AverageEnergy <span style="color: #666666">+=</span> Energy
AverageEnergy2 <span style="color: #666666">+=</span> Energy<span style="color: #666666">*</span>Energy
<span style="color: #408080; font-style: italic">#Final averages</span>
AverageEnergy <span style="color: #666666">=</span> AverageEnergy<span style="color: #666666">/</span>MCcycles
AverageEnergy2 <span style="color: #666666">=</span> AverageEnergy2<span style="color: #666666">/</span>MCcycles
Variance <span style="color: #666666">=</span> AverageEnergy2 <span style="color: #666666">-</span> AverageEnergy<span style="color: #666666">*</span>AverageEnergy
<span style="color: #008000; font-weight: bold">print</span>(AverageEnergy, Variance)
n, bins, patches <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>hist(VelocityValues, <span style="color: #666666">400</span>, facecolor<span style="color: #666666">=</span><span style="color: #BA2121">&#39;green&#39;</span>)
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&#39;$v$&#39;</span>)
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">&#39;Velocity distribution P(v)&#39;</span>)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">r&#39;Velocity histogram at $k_BT=2$&#39;</span>)
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">-5</span>, <span style="color: #666666">5</span>, <span style="color: #666666">0</span>, <span style="color: #666666">600</span>])
plt<span style="color: #666666">.</span>grid(<span style="color: #008000">True</span>)
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">collections</span> <span style="color: #008000; font-weight: bold">import</span> Counter
<span style="color: #408080; font-style: italic">#print (Counter(VelocityValues))</span>
<span style="color: #008000; font-weight: bold">print</span> (VelocityValues[:<span style="color: #666666">20</span>])
VelocityValues<span style="color: #666666">=</span><span style="color: #008000">list</span>(Counter(VelocityValues)<span style="color: #666666">.</span>keys())
d<span style="color: #666666">=</span><span style="color: #008000">list</span>(Counter(VelocityValues)<span style="color: #666666">.</span>values())
VelocityValues<span style="color: #666666">=</span>np<span style="color: #666666">.</span>asarray(VelocityValues)[:, np<span style="color: #666666">.</span>newaxis]
d<span style="color: #666666">=</span>np<span style="color: #666666">.</span>asarray(d)
<span style="color: #008000; font-weight: bold">print</span> (VelocityValues<span style="color: #666666">.</span>shape, d<span style="color: #666666">.</span>shape)
plt<span style="color: #666666">.</span>scatter(VelocityValues, d)
plt<span style="color: #666666">.</span>show()
<span style="color: #408080; font-style: italic">#2nd Degree Polynomial</span>
poly_feat<span style="color: #666666">=</span>PolynomialFeatures(degree<span style="color: #666666">=20</span>, include_bias<span style="color: #666666">=</span><span style="color: #008000">False</span>)
X_poly<span style="color: #666666">=</span>poly_feat<span style="color: #666666">.</span>fit_transform(VelocityValues)
lin_reg<span style="color: #666666">=</span>LinearRegression()
poly_fit<span style="color: #666666">=</span>lin_reg<span style="color: #666666">.</span>fit(X_poly,d)
y_plot<span style="color: #666666">=</span>poly_fit<span style="color: #666666">.</span>predict(X_poly)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;Polynomial Fit&quot;</span>)
plt<span style="color: #666666">.</span>plot(VelocityValues, y_plot, color<span style="color: #666666">=</span><span style="color: #BA2121">&#39;black&#39;</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;Fit&quot;</span>)
plt<span style="color: #666666">.</span>show()
<span style="color: #408080; font-style: italic">#Decision Trees</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeRegressor
regr_1<span style="color: #666666">=</span>DecisionTreeRegressor(max_depth<span style="color: #666666">=2</span>)
regr_2<span style="color: #666666">=</span>DecisionTreeRegressor(max_depth<span style="color: #666666">=5</span>)
regr_3<span style="color: #666666">=</span>DecisionTreeRegressor(max_depth<span style="color: #666666">=7</span>)
regr_1<span style="color: #666666">.</span>fit(VelocityValues, d)
regr_2<span style="color: #666666">.</span>fit(VelocityValues, d)
regr_3<span style="color: #666666">.</span>fit(VelocityValues, d)
X_test <span style="color: #666666">=</span> np<span style="color: #666666">.</span>arange(<span style="color: #666666">0.0</span>, MCcycles, <span style="color: #666666">0.01</span>)[:, np<span style="color: #666666">.</span>newaxis]
y_1<span style="color: #666666">=</span>regr_1<span style="color: #666666">.</span>predict(X_test)
y_2<span style="color: #666666">=</span>regr_2<span style="color: #666666">.</span>predict(X_test)
y_3<span style="color: #666666">=</span>regr_3<span style="color: #666666">.</span>predict(X_test)
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;Decision Tree&quot;</span>)
plt<span style="color: #666666">.</span>plot(X_test, y_1, color<span style="color: #666666">=</span><span style="color: #BA2121">&quot;red&quot;</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;max_depth=2&quot;</span>, linewidth<span style="color: #666666">=2</span>)
plt<span style="color: #666666">.</span>plot(X_test, y_2, color<span style="color: #666666">=</span><span style="color: #BA2121">&quot;green&quot;</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;max_depth=5&quot;</span>, linewidth<span style="color: #666666">=2</span>)
plt<span style="color: #666666">.</span>plot(X_test, y_3, color<span style="color: #666666">=</span><span style="color: #BA2121">&quot;m&quot;</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">&quot;max_depth=7&quot;</span>, linewidth<span style="color: #666666">=2</span>)
plt<span style="color: #666666">.</span>show()
</pre></div>
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec4">Building a tree, regression </h2>
<h2 id="___sec3">Building a tree, regression </h2>
<p>
There are mainly two steps
@@ -429,7 +327,7 @@ box.
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec5">A top-down approach, recursive binary splitting </h2>
<h2 id="___sec4">A top-down approach, recursive binary splitting </h2>
<p>
Unfortunately, it is computationally infeasible to consider every
@@ -448,7 +346,7 @@ better tree in some future step.
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec6">Making a tree </h2>
<h2 id="___sec5">Making a tree </h2>
<p>
In order to implement the recursive binary splitting we start by selecting
@@ -495,7 +393,7 @@ region contains more than five observations.
<p>
<!-- !split -->
<h2 id="___sec7">Pruning the tree </h2>
<h2 id="___sec6">Pruning the tree </h2>
<p>
The above procedure is rather straightforward, but leads often to
@@ -514,7 +412,7 @@ parameter \( \alpha \).
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec8">Cost complexity pruning </h2>
<h2 id="___sec7">Cost complexity pruning </h2>
For each value of \( \alpha \) there corresponds a subtree \( T \in T_0 \) such that
$$
\sum_{m=1}^{\overline{T}}\sum_{i:x_i\in R_m}(y_i-\overline{y}_{R_m})^2+\alpha\overline{T},
@@ -545,7 +443,7 @@ subtree corresponding to \( \alpha \).
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec9">A schematic procedure </h2>
<h2 id="___sec8">A schematic procedure </h2>
<p>
<div class="alert alert-block alert-block alert-text-normal">
@@ -571,7 +469,7 @@ subtree corresponding to \( \alpha \).
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec10">A classification tree </h2>
<h2 id="___sec9">A classification tree </h2>
<p>
A classification tree is very similar to a regression tree, except
@@ -590,7 +488,7 @@ fall into that region.
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec11">Growing a classification tree </h2>
<h2 id="___sec10">Growing a classification tree </h2>
<p>
The task of growing a
@@ -614,7 +512,7 @@ than is the classification error rate.
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec12">Classification tree, how to split nodes </h2>
<h2 id="___sec11">Classification tree, how to split nodes </h2>
If our targets are the outcome of a classification process that takes for example
\( k=1,2,\dots,K \) values, the only thing we need to think of is to set up the splitting criteria for each node.
@@ -656,12 +554,15 @@ $$
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec13">Entropy and the ID3 algorithm </h2>
<h2 id="___sec12">Entropy and the ID3 algorithm </h2>
<p>
More text to come here.
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec14">Writing your own code for a classification tree </h2>
<h2 id="___sec13">Writing your own code for a classification tree </h2>
<p>
@@ -753,7 +654,7 @@ $$
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec15">Back to moons again </h2>
<h2 id="___sec14">Back to moons again </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
@@ -825,7 +726,7 @@ plt<span style="color: #666666">.</span>show()
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec16">Playing around with regions </h2>
<h2 id="___sec15">Playing around with regions </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
@@ -853,7 +754,7 @@ plt<span style="color: #666666">.</span>show()
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec17">Regression trees </h2>
<h2 id="___sec16">Regression trees </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
@@ -875,7 +776,7 @@ tree_reg<span style="color: #666666">.</span>fit(X, y)
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec18">Final regressor code </h2>
<h2 id="___sec17">Final regressor code </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
@@ -953,7 +854,7 @@ plt<span style="color: #666666">.</span>show()
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec19">Classification again: The zoo data </h2>
<h2 id="___sec18">Classification again: The zoo data </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
@@ -981,7 +882,7 @@ prediction <span style="color: #666666">=</span> tree<span style="color: #666666
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec20">Pros and cons of trees, pros </h2>
<h2 id="___sec19">Pros and cons of trees, pros </h2>
<ul>
<li> White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines)</li>
@@ -995,7 +896,7 @@ prediction <span style="color: #666666">=</span> tree<span style="color: #666666
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec21">Disadvantages </h2>
<h2 id="___sec20">Disadvantages </h2>
<ul>
<li> Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches</li>
@@ -1012,7 +913,7 @@ However, by aggregating many decision trees, using methods like bagging, random
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec22">Bagging </h2>
<h2 id="___sec21">Bagging </h2>
<p>
The <b>plain</b> decision trees suffer from high
@@ -1055,7 +956,7 @@ predictor, averaged over all \( B \) trees.
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec23">Simple example, head or tail </h2>
<h2 id="___sec22">Simple example, head or tail </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
@@ -1075,7 +976,7 @@ plt<span style="color: #666666">.</span>show()
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec24">Random forests </h2>
<h2 id="___sec23">Random forests </h2>
<p>
Random forests provide an improvement over bagged trees by way of a
@@ -1117,7 +1018,7 @@ setting.
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec25">A simple scikit-learn example </h2>
<h2 id="___sec24">A simple scikit-learn example </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
@@ -1135,7 +1036,7 @@ accuracy <span style="color: #666666">=</span> cross_validate(Random_Forest_mode
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec26">Please, not the moons again! </h2>
<h2 id="___sec25">Please, not the moons again! </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
@@ -1193,7 +1094,7 @@ voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec27">Bagging examples </h2>
<h2 id="___sec26">Bagging examples </h2>
<p>
@@ -1254,7 +1155,7 @@ plt<span style="color: #666666">.</span>show()
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="___sec28">Then random forests </h2>
<h2 id="___sec27">Then random forests </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
@@ -1276,7 +1177,7 @@ np<span style="color: #666666">.</span>sum(y_pred <span style="color: #666666">=
<p>
<!-- !split -->
<h2 id="___sec29">Boosting and more </h2>
<h2 id="___sec28">Boosting and more </h2>
More material to come here.
<p>
@@ -1285,7 +1186,7 @@ More material to come here.
<center style="font-size:80%">
<!-- copyright --> &copy; 1999-2018, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
<!-- copyright --> &copy; 1999-2019, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
</center>
doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
+88 -201
View File
File diff suppressed because one or more lines are too long
doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz
BIN
View File
Binary file not shown.
doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf
BIN
View File
Binary file not shown.
doc/src/DecisionTrees/DecisionTrees.do.txt
+1 -98
View File
@@ -149,104 +149,6 @@ plt.show()
!ec
!split
===== Maxwell-Boltzmann velocity distribution =====
!bc pycod
# Program to test the Metropolis algorithm with one particle at given temp in
# one dimension
#!/usr/bin/env python
import numpy as np
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
import random
from math import sqrt, exp, log
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
# initialize the rng with a seed
random.seed()
# Hard coding of input parameters
MCcycles = 100000
Temperature = 2.0
beta = 1./Temperature
InitialVelocity = -2.0
CurrentVelocity = InitialVelocity
Energy = 0.5*InitialVelocity*InitialVelocity
VelocityRange = 10*sqrt(Temperature)
VelocityStep = 2*VelocityRange/10.
AverageEnergy = Energy
AverageEnergy2 = Energy*Energy
VelocityValues = np.zeros(MCcycles)
# The Monte Carlo sampling with Metropolis starts here
for i in range (1, MCcycles, 1):
TrialVelocity = CurrentVelocity + (2.0*random.random() - 1.0)*VelocityStep
EnergyChange = 0.5*(TrialVelocity*TrialVelocity -CurrentVelocity*CurrentVelocity);
if random.random() <= exp(-beta*EnergyChange):
CurrentVelocity = TrialVelocity
Energy += EnergyChange
VelocityValues[i] = CurrentVelocity
AverageEnergy += Energy
AverageEnergy2 += Energy*Energy
#Final averages
AverageEnergy = AverageEnergy/MCcycles
AverageEnergy2 = AverageEnergy2/MCcycles
Variance = AverageEnergy2 - AverageEnergy*AverageEnergy
print(AverageEnergy, Variance)
n, bins, patches = plt.hist(VelocityValues, 400, facecolor='green')
plt.xlabel('$v$')
plt.ylabel('Velocity distribution P(v)')
plt.title(r'Velocity histogram at $k_BT=2$')
plt.axis([-5, 5, 0, 600])
plt.grid(True)
from collections import Counter
#print (Counter(VelocityValues))
print (VelocityValues[:20])
VelocityValues=list(Counter(VelocityValues).keys())
d=list(Counter(VelocityValues).values())
VelocityValues=np.asarray(VelocityValues)[:, np.newaxis]
d=np.asarray(d)
print (VelocityValues.shape, d.shape)
plt.scatter(VelocityValues, d)
plt.show()
#2nd Degree Polynomial
poly_feat=PolynomialFeatures(degree=20, include_bias=False)
X_poly=poly_feat.fit_transform(VelocityValues)
lin_reg=LinearRegression()
poly_fit=lin_reg.fit(X_poly,d)
y_plot=poly_fit.predict(X_poly)
plt.title("Polynomial Fit")
plt.plot(VelocityValues, y_plot, color='black', label="Fit")
plt.show()
#Decision Trees
from sklearn.tree import DecisionTreeRegressor
regr_1=DecisionTreeRegressor(max_depth=2)
regr_2=DecisionTreeRegressor(max_depth=5)
regr_3=DecisionTreeRegressor(max_depth=7)
regr_1.fit(VelocityValues, d)
regr_2.fit(VelocityValues, d)
regr_3.fit(VelocityValues, d)
X_test = np.arange(0.0, MCcycles, 0.01)[:, np.newaxis]
y_1=regr_1.predict(X_test)
y_2=regr_2.predict(X_test)
y_3=regr_3.predict(X_test)
plt.title("Decision Tree")
plt.plot(X_test, y_1, color="red", label="max_depth=2", linewidth=2)
plt.plot(X_test, y_2, color="green", label="max_depth=5", linewidth=2)
plt.plot(X_test, y_3, color="m", label="max_depth=7", linewidth=2)
plt.show()
!ec
!split
@@ -463,6 +365,7 @@ s = -\sum_{k=1}^K p_{mk}\log{p_{mk}}.
!split
===== Entropy and the ID3 algorithm =====
More text to come here.
!split
===== Writing your own code for a classification tree =====