update on decision trees
This commit is contained in:
mhjensen
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -195,7 +202,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Oct 29, 2019</h4></center> <!-- date -->
|
||||
<center><h4>Oct 31, 2019</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
|
||||
@@ -219,7 +226,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._DecisionTrees-bs008.html">9</a></li>
|
||||
<li><a href="._DecisionTrees-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -224,7 +231,7 @@ given some assumptions, make predictions about the target feature value
|
||||
<li><a href="._DecisionTrees-bs009.html">10</a></li>
|
||||
<li><a href="._DecisionTrees-bs010.html">11</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs002.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -199,7 +206,7 @@ In the figure here we present a decision tree obtained from a classification pro
|
||||
<li><a href="._DecisionTrees-bs010.html">11</a></li>
|
||||
<li><a href="._DecisionTrees-bs011.html">12</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs003.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -200,7 +207,7 @@ In the figure we present a decision tree obtained from a simple regression prob
|
||||
<li><a href="._DecisionTrees-bs011.html">12</a></li>
|
||||
<li><a href="._DecisionTrees-bs012.html">13</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs004.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -211,7 +218,7 @@ node.
|
||||
<li><a href="._DecisionTrees-bs012.html">13</a></li>
|
||||
<li><a href="._DecisionTrees-bs013.html">14</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs005.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -212,7 +219,7 @@ Then we are essentially done!
|
||||
<li><a href="._DecisionTrees-bs013.html">14</a></li>
|
||||
<li><a href="._DecisionTrees-bs014.html">15</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs006.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -291,7 +298,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._DecisionTrees-bs014.html">15</a></li>
|
||||
<li><a href="._DecisionTrees-bs015.html">16</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs007.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -224,7 +231,7 @@ within box \( j \).
|
||||
<li><a href="._DecisionTrees-bs015.html">16</a></li>
|
||||
<li><a href="._DecisionTrees-bs016.html">17</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs008.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -216,7 +223,7 @@ better tree in some future step.
|
||||
<li><a href="._DecisionTrees-bs016.html">17</a></li>
|
||||
<li><a href="._DecisionTrees-bs017.html">18</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs009.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -249,7 +256,7 @@ region contains more than five observations.
|
||||
<li><a href="._DecisionTrees-bs017.html">18</a></li>
|
||||
<li><a href="._DecisionTrees-bs018.html">19</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs010.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -218,7 +225,7 @@ parameter \( \alpha \).
|
||||
<li><a href="._DecisionTrees-bs018.html">19</a></li>
|
||||
<li><a href="._DecisionTrees-bs019.html">20</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs011.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -230,7 +237,7 @@ subtree corresponding to \( \alpha \).
|
||||
<li><a href="._DecisionTrees-bs019.html">20</a></li>
|
||||
<li><a href="._DecisionTrees-bs020.html">21</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs012.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -226,7 +233,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._DecisionTrees-bs020.html">21</a></li>
|
||||
<li><a href="._DecisionTrees-bs021.html">22</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs013.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -218,7 +225,7 @@ fall into that region.
|
||||
<li><a href="._DecisionTrees-bs021.html">22</a></li>
|
||||
<li><a href="._DecisionTrees-bs022.html">23</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs014.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -223,7 +230,7 @@ than is the classification error rate.
|
||||
<li><a href="._DecisionTrees-bs022.html">23</a></li>
|
||||
<li><a href="._DecisionTrees-bs023.html">24</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs015.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -249,7 +256,7 @@ $$
|
||||
<li><a href="._DecisionTrees-bs023.html">24</a></li>
|
||||
<li><a href="._DecisionTrees-bs024.html">25</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs016.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -176,185 +183,43 @@ MathJax.Hub.Config({
|
||||
<a name="part0016"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec15" class="anchor">The CART (Classification and Regression Tree) algorithm </h2>
|
||||
|
||||
<p>
|
||||
The above functions (gini, entropy and misclassification error) are important components of the so-called CART algorithm. We will discuss this algorithm first before we move on to the information gain algorithm ID3.
|
||||
|
||||
<h2 id="___sec15" class="anchor">Visualizing the Tree, Classification </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> seed
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> randrange
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">csv</span> <span style="color: #008000; font-weight: bold">import</span> reader
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Load a CSV file</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">load_csv</span>(filename):
|
||||
<span style="color: #008000">file</span> <span style="color: #666666">=</span> <span style="color: #008000">open</span>(filename, <span style="color: #BA2121">"rb"</span>)
|
||||
lines <span style="color: #666666">=</span> reader(<span style="color: #008000">file</span>)
|
||||
dataset <span style="color: #666666">=</span> <span style="color: #008000">list</span>(lines)
|
||||
<span style="color: #008000; font-weight: bold">return</span> dataset
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Convert string column to float</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">str_column_to_float</span>(dataset, column):
|
||||
<span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> dataset:
|
||||
row[column] <span style="color: #666666">=</span> <span style="color: #008000">float</span>(row[column]<span style="color: #666666">.</span>strip())
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Split a dataset into k folds</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">cross_validation_split</span>(dataset, n_folds):
|
||||
dataset_split <span style="color: #666666">=</span> <span style="color: #008000">list</span>()
|
||||
dataset_copy <span style="color: #666666">=</span> <span style="color: #008000">list</span>(dataset)
|
||||
fold_size <span style="color: #666666">=</span> <span style="color: #008000">int</span>(<span style="color: #008000">len</span>(dataset) <span style="color: #666666">/</span> n_folds)
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_folds):
|
||||
fold <span style="color: #666666">=</span> <span style="color: #008000">list</span>()
|
||||
<span style="color: #008000; font-weight: bold">while</span> <span style="color: #008000">len</span>(fold) <span style="color: #666666"><</span> fold_size:
|
||||
index <span style="color: #666666">=</span> randrange(<span style="color: #008000">len</span>(dataset_copy))
|
||||
fold<span style="color: #666666">.</span>append(dataset_copy<span style="color: #666666">.</span>pop(index))
|
||||
dataset_split<span style="color: #666666">.</span>append(fold)
|
||||
<span style="color: #008000; font-weight: bold">return</span> dataset_split
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Calculate accuracy percentage</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">accuracy_metric</span>(actual, predicted):
|
||||
correct <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(actual)):
|
||||
<span style="color: #008000; font-weight: bold">if</span> actual[i] <span style="color: #666666">==</span> predicted[i]:
|
||||
correct <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
||||
<span style="color: #008000; font-weight: bold">return</span> correct <span style="color: #666666">/</span> <span style="color: #008000">float</span>(<span style="color: #008000">len</span>(actual)) <span style="color: #666666">*</span> <span style="color: #666666">100.0</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Evaluate an algorithm using a cross validation split</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">evaluate_algorithm</span>(dataset, algorithm, n_folds, <span style="color: #666666">*</span>args):
|
||||
folds <span style="color: #666666">=</span> cross_validation_split(dataset, n_folds)
|
||||
scores <span style="color: #666666">=</span> <span style="color: #008000">list</span>()
|
||||
<span style="color: #008000; font-weight: bold">for</span> fold <span style="color: #AA22FF; font-weight: bold">in</span> folds:
|
||||
train_set <span style="color: #666666">=</span> <span style="color: #008000">list</span>(folds)
|
||||
train_set<span style="color: #666666">.</span>remove(fold)
|
||||
train_set <span style="color: #666666">=</span> <span style="color: #008000">sum</span>(train_set, [])
|
||||
test_set <span style="color: #666666">=</span> <span style="color: #008000">list</span>()
|
||||
<span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> fold:
|
||||
row_copy <span style="color: #666666">=</span> <span style="color: #008000">list</span>(row)
|
||||
test_set<span style="color: #666666">.</span>append(row_copy)
|
||||
row_copy[<span style="color: #666666">-1</span>] <span style="color: #666666">=</span> <span style="color: #008000">None</span>
|
||||
predicted <span style="color: #666666">=</span> algorithm(train_set, test_set, <span style="color: #666666">*</span>args)
|
||||
actual <span style="color: #666666">=</span> [row[<span style="color: #666666">-1</span>] <span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> fold]
|
||||
accuracy <span style="color: #666666">=</span> accuracy_metric(actual, predicted)
|
||||
scores<span style="color: #666666">.</span>append(accuracy)
|
||||
<span style="color: #008000; font-weight: bold">return</span> scores
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Split a dataset based on an attribute and an attribute value</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">test_split</span>(index, value, dataset):
|
||||
left, right <span style="color: #666666">=</span> <span style="color: #008000">list</span>(), <span style="color: #008000">list</span>()
|
||||
<span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> dataset:
|
||||
<span style="color: #008000; font-weight: bold">if</span> row[index] <span style="color: #666666"><</span> value:
|
||||
left<span style="color: #666666">.</span>append(row)
|
||||
<span style="color: #008000; font-weight: bold">else</span>:
|
||||
right<span style="color: #666666">.</span>append(row)
|
||||
<span style="color: #008000; font-weight: bold">return</span> left, right
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Calculate the Gini index for a split dataset</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">gini_index</span>(groups, classes):
|
||||
<span style="color: #408080; font-style: italic"># count all samples at split point</span>
|
||||
n_instances <span style="color: #666666">=</span> <span style="color: #008000">float</span>(<span style="color: #008000">sum</span>([<span style="color: #008000">len</span>(group) <span style="color: #008000; font-weight: bold">for</span> group <span style="color: #AA22FF; font-weight: bold">in</span> groups]))
|
||||
<span style="color: #408080; font-style: italic"># sum weighted Gini index for each group</span>
|
||||
gini <span style="color: #666666">=</span> <span style="color: #666666">0.0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> group <span style="color: #AA22FF; font-weight: bold">in</span> groups:
|
||||
size <span style="color: #666666">=</span> <span style="color: #008000">float</span>(<span style="color: #008000">len</span>(group))
|
||||
<span style="color: #408080; font-style: italic"># avoid divide by zero</span>
|
||||
<span style="color: #008000; font-weight: bold">if</span> size <span style="color: #666666">==</span> <span style="color: #666666">0</span>:
|
||||
<span style="color: #008000; font-weight: bold">continue</span>
|
||||
score <span style="color: #666666">=</span> <span style="color: #666666">0.0</span>
|
||||
<span style="color: #408080; font-style: italic"># score the group based on the score for each class</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> class_val <span style="color: #AA22FF; font-weight: bold">in</span> classes:
|
||||
p <span style="color: #666666">=</span> [row[<span style="color: #666666">-1</span>] <span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> group]<span style="color: #666666">.</span>count(class_val) <span style="color: #666666">/</span> size
|
||||
score <span style="color: #666666">+=</span> p <span style="color: #666666">*</span> p
|
||||
<span style="color: #408080; font-style: italic"># weight the group score by its relative size</span>
|
||||
gini <span style="color: #666666">+=</span> (<span style="color: #666666">1.0</span> <span style="color: #666666">-</span> score) <span style="color: #666666">*</span> (size <span style="color: #666666">/</span> n_instances)
|
||||
<span style="color: #008000; font-weight: bold">return</span> gini
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Select the best split point for a dataset</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">get_split</span>(dataset):
|
||||
class_values <span style="color: #666666">=</span> <span style="color: #008000">list</span>(<span style="color: #008000">set</span>(row[<span style="color: #666666">-1</span>] <span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> dataset))
|
||||
b_index, b_value, b_score, b_groups <span style="color: #666666">=</span> <span style="color: #666666">999</span>, <span style="color: #666666">999</span>, <span style="color: #666666">999</span>, <span style="color: #008000">None</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> index <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(dataset[<span style="color: #666666">0</span>])<span style="color: #666666">-1</span>):
|
||||
<span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> dataset:
|
||||
groups <span style="color: #666666">=</span> test_split(index, row[index], dataset)
|
||||
gini <span style="color: #666666">=</span> gini_index(groups, class_values)
|
||||
<span style="color: #008000; font-weight: bold">if</span> gini <span style="color: #666666"><</span> b_score:
|
||||
b_index, b_value, b_score, b_groups <span style="color: #666666">=</span> index, row[index], gini, groups
|
||||
<span style="color: #008000; font-weight: bold">return</span> {<span style="color: #BA2121">'index'</span>:b_index, <span style="color: #BA2121">'value'</span>:b_value, <span style="color: #BA2121">'groups'</span>:b_groups}
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Create a terminal node value</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">to_terminal</span>(group):
|
||||
outcomes <span style="color: #666666">=</span> [row[<span style="color: #666666">-1</span>] <span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> group]
|
||||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">max</span>(<span style="color: #008000">set</span>(outcomes), key<span style="color: #666666">=</span>outcomes<span style="color: #666666">.</span>count)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Create child splits for a node or make terminal</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">split</span>(node, max_depth, min_size, depth):
|
||||
left, right <span style="color: #666666">=</span> node[<span style="color: #BA2121">'groups'</span>]
|
||||
<span style="color: #008000; font-weight: bold">del</span>(node[<span style="color: #BA2121">'groups'</span>])
|
||||
<span style="color: #408080; font-style: italic"># check for a no split</span>
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> left <span style="color: #AA22FF; font-weight: bold">or</span> <span style="color: #AA22FF; font-weight: bold">not</span> right:
|
||||
node[<span style="color: #BA2121">'left'</span>] <span style="color: #666666">=</span> node[<span style="color: #BA2121">'right'</span>] <span style="color: #666666">=</span> to_terminal(left <span style="color: #666666">+</span> right)
|
||||
<span style="color: #008000; font-weight: bold">return</span>
|
||||
<span style="color: #408080; font-style: italic"># check for max depth</span>
|
||||
<span style="color: #008000; font-weight: bold">if</span> depth <span style="color: #666666">>=</span> max_depth:
|
||||
node[<span style="color: #BA2121">'left'</span>], node[<span style="color: #BA2121">'right'</span>] <span style="color: #666666">=</span> to_terminal(left), to_terminal(right)
|
||||
<span style="color: #008000; font-weight: bold">return</span>
|
||||
<span style="color: #408080; font-style: italic"># process left child</span>
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">len</span>(left) <span style="color: #666666"><=</span> min_size:
|
||||
node[<span style="color: #BA2121">'left'</span>] <span style="color: #666666">=</span> to_terminal(left)
|
||||
<span style="color: #008000; font-weight: bold">else</span>:
|
||||
node[<span style="color: #BA2121">'left'</span>] <span style="color: #666666">=</span> get_split(left)
|
||||
split(node[<span style="color: #BA2121">'left'</span>], max_depth, min_size, depth<span style="color: #666666">+1</span>)
|
||||
<span style="color: #408080; font-style: italic"># process right child</span>
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">len</span>(right) <span style="color: #666666"><=</span> min_size:
|
||||
node[<span style="color: #BA2121">'right'</span>] <span style="color: #666666">=</span> to_terminal(right)
|
||||
<span style="color: #008000; font-weight: bold">else</span>:
|
||||
node[<span style="color: #BA2121">'right'</span>] <span style="color: #666666">=</span> get_split(right)
|
||||
split(node[<span style="color: #BA2121">'right'</span>], max_depth, min_size, depth<span style="color: #666666">+1</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Build a decision tree</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">build_tree</span>(train, max_depth, min_size):
|
||||
root <span style="color: #666666">=</span> get_split(train)
|
||||
split(root, max_depth, min_size, <span style="color: #666666">1</span>)
|
||||
<span style="color: #008000; font-weight: bold">return</span> root
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Make a prediction with a decision tree</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">predict</span>(node, row):
|
||||
<span style="color: #008000; font-weight: bold">if</span> row[node[<span style="color: #BA2121">'index'</span>]] <span style="color: #666666"><</span> node[<span style="color: #BA2121">'value'</span>]:
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">isinstance</span>(node[<span style="color: #BA2121">'left'</span>], <span style="color: #008000">dict</span>):
|
||||
<span style="color: #008000; font-weight: bold">return</span> predict(node[<span style="color: #BA2121">'left'</span>], row)
|
||||
<span style="color: #008000; font-weight: bold">else</span>:
|
||||
<span style="color: #008000; font-weight: bold">return</span> node[<span style="color: #BA2121">'left'</span>]
|
||||
<span style="color: #008000; font-weight: bold">else</span>:
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">isinstance</span>(node[<span style="color: #BA2121">'right'</span>], <span style="color: #008000">dict</span>):
|
||||
<span style="color: #008000; font-weight: bold">return</span> predict(node[<span style="color: #BA2121">'right'</span>], row)
|
||||
<span style="color: #008000; font-weight: bold">else</span>:
|
||||
<span style="color: #008000; font-weight: bold">return</span> node[<span style="color: #BA2121">'right'</span>]
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Classification and Regression Tree Algorithm</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">decision_tree</span>(train, test, max_depth, min_size):
|
||||
tree <span style="color: #666666">=</span> build_tree(train, max_depth, min_size)
|
||||
predictions <span style="color: #666666">=</span> <span style="color: #008000">list</span>()
|
||||
<span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> test:
|
||||
prediction <span style="color: #666666">=</span> predict(tree, row)
|
||||
predictions<span style="color: #666666">.</span>append(prediction)
|
||||
<span style="color: #008000; font-weight: bold">return</span>(predictions)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Test CART </span>
|
||||
seed(<span style="color: #666666">1</span>)
|
||||
<span style="color: #408080; font-style: italic"># load and prepare data</span>
|
||||
filename <span style="color: #666666">=</span> <span style="color: #BA2121">'DataFiles/rideclass.csv'</span>
|
||||
dataset <span style="color: #666666">=</span> load_csv(filename)
|
||||
<span style="color: #408080; font-style: italic"># convert string attributes to integers</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(dataset[<span style="color: #666666">0</span>])):
|
||||
str_column_to_float(dataset, i)
|
||||
<span style="color: #408080; font-style: italic"># evaluate algorithm</span>
|
||||
n_folds <span style="color: #666666">=</span> <span style="color: #666666">5</span>
|
||||
max_depth <span style="color: #666666">=</span> <span style="color: #666666">5</span>
|
||||
min_size <span style="color: #666666">=</span> <span style="color: #666666">10</span>
|
||||
scores <span style="color: #666666">=</span> evaluate_algorithm(dataset, decision_tree, n_folds, max_depth, min_size)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'Scores: </span><span style="color: #BB6688; font-weight: bold">%s</span><span style="color: #BA2121">'</span> <span style="color: #666666">%</span> scores)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'Mean Accuracy: </span><span style="color: #BB6688; font-weight: bold">%.3f%%</span><span style="color: #BA2121">'</span> <span style="color: #666666">%</span> (<span style="color: #008000">sum</span>(scores)<span style="color: #666666">/</span><span style="color: #008000">float</span>(<span style="color: #008000">len</span>(scores))))
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> confusion_matrix
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> export_graphviz
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> Image
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">pydot</span> <span style="color: #008000; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
|
||||
|
||||
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
||||
X <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>DataFrame(cancer<span style="color: #666666">.</span>data, columns<span style="color: #666666">=</span>cancer<span style="color: #666666">.</span>feature_names)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(X)
|
||||
y <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>Categorical<span style="color: #666666">.</span>from_codes(cancer<span style="color: #666666">.</span>target, cancer<span style="color: #666666">.</span>target_names)
|
||||
y <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>get_dummies(y)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(y)
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(X, y, random_state<span style="color: #666666">=1</span>)
|
||||
tree_clf <span style="color: #666666">=</span> DecisionTreeClassifier(max_depth<span style="color: #666666">=5</span>)
|
||||
tree_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file<span style="color: #666666">=</span><span style="color: #BA2121">"DataFiles/cancer.dot"</span>,
|
||||
feature_names<span style="color: #666666">=</span>cancer<span style="color: #666666">.</span>feature_names,
|
||||
class_names<span style="color: #666666">=</span>cancer<span style="color: #666666">.</span>target_names,
|
||||
rounded<span style="color: #666666">=</span><span style="color: #008000">True</span>,
|
||||
filled<span style="color: #666666">=</span><span style="color: #008000">True</span>
|
||||
)
|
||||
cmd <span style="color: #666666">=</span> <span style="color: #BA2121">'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'</span>
|
||||
os<span style="color: #666666">.</span>system(cmd)
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
@@ -382,7 +247,7 @@ scores <span style="color: #666666">=</span> evaluate_algorithm(dataset, decisio
|
||||
<li><a href="._DecisionTrees-bs024.html">25</a></li>
|
||||
<li><a href="._DecisionTrees-bs025.html">26</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs017.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -176,39 +183,35 @@ MathJax.Hub.Config({
|
||||
<a name="part0017"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec16" class="anchor">Entropy and the ID3 algorithm </h2>
|
||||
|
||||
<h2 id="___sec16" class="anchor">Visualizing the Tree, The Moons </h2>
|
||||
<p>
|
||||
ID3, learns decision trees by constructing
|
||||
them topdown, beginning with the question <b>which attribute should be tested at the root of the tree</b>?
|
||||
|
||||
<ol>
|
||||
<li> Each instance attribute is evaluated using a statistical test to determine how well it alone classifies the training examples.</li>
|
||||
<li> The best attribute is selected and used as the test at the root node of the tree.</li>
|
||||
<li> A descendant of the root node is then created for each possible value of this attribute.</li>
|
||||
<li> Training examples are sorted to the appropriate descendant node.</li>
|
||||
<li> The entire process is then repeated using the training examples associated with each descendant node to select the best attribute to test at that point in the tree.</li>
|
||||
<li> This forms a greedy search for an acceptable decision tree, in which the algorithm never backtracks to reconsider earlier choices.</li>
|
||||
</ol>
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Common imports</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_moons
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> export_graphviz
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">pydot</span> <span style="color: #008000; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
||||
|
||||
The ID3 algorithm selects, which attribute to test at each node in the
|
||||
tree.
|
||||
|
||||
<p>
|
||||
We would like to select the attribute that is most useful for classifying
|
||||
examples.
|
||||
|
||||
<p>
|
||||
What is a good quantitative measure of the worth of an attribute?
|
||||
|
||||
<p>
|
||||
Information gain measures how well a given attribute separates the
|
||||
training examples according to their target classification.
|
||||
|
||||
<p>
|
||||
The ID3 algorithm uses this information gain measure to select among the candidate
|
||||
attributes at each step while growing the tree.
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">42</span>)
|
||||
X, y <span style="color: #666666">=</span> make_moons(n_samples<span style="color: #666666">=100</span>, noise<span style="color: #666666">=0.25</span>, random_state<span style="color: #666666">=53</span>)
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(X,y,random_state<span style="color: #666666">=0</span>)
|
||||
tree_clf <span style="color: #666666">=</span> DecisionTreeClassifier(max_depth<span style="color: #666666">=5</span>)
|
||||
tree_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file<span style="color: #666666">=</span><span style="color: #BA2121">"DataFiles/moons.dot"</span>,
|
||||
rounded<span style="color: #666666">=</span><span style="color: #008000">True</span>,
|
||||
filled<span style="color: #666666">=</span><span style="color: #008000">True</span>
|
||||
)
|
||||
cmd <span style="color: #666666">=</span> <span style="color: #BA2121">'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png'</span>
|
||||
os<span style="color: #666666">.</span>system(cmd)
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -235,7 +238,7 @@ attributes at each step while growing the tree.
|
||||
<li><a href="._DecisionTrees-bs025.html">26</a></li>
|
||||
<li><a href="._DecisionTrees-bs026.html">27</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs018.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -176,11 +183,48 @@ MathJax.Hub.Config({
|
||||
<a name="part0018"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec17" class="anchor">Implementing the ID3 Algorithm </h2>
|
||||
<h2 id="___sec17" class="anchor">Computing the Gini index </h2>
|
||||
|
||||
<p>
|
||||
<b>more text to come here</b>, material presented during lecture Friday Oct 25.
|
||||
The example we will look at is a classical one in many Machine
|
||||
Learning applications. Based on various meteorological features, we
|
||||
have several so-called attributes which decide whether we at the end
|
||||
will do some outdoor activity like skiing, going for a bike ride etc
|
||||
etc. The table here contains the feautures <b>outlook</b>, <b>temperature</b>,
|
||||
<b>humidity</b> and <b>wind</b>. The target or output is whether we ride
|
||||
(True=1) or whether we do something else that day (False=0). The
|
||||
attributes for each feature are then sunny, overcast and rain for the
|
||||
outlook, hot, cold and mild for temperature, high and normal for
|
||||
humidity and weak and strong for wind.
|
||||
|
||||
<p>
|
||||
The table here summarizes the various attributes and
|
||||
|
||||
<div class="row">
|
||||
<div class="col-xs-4">
|
||||
<table class="table table-striped table-hover table-condensed">
|
||||
<thead>
|
||||
<tr><td align="center"><b>Day</b></td> <td align="center"><b>Outlook </b></td> <td align="center"><b>Temperature</b></td> <td align="center"><b>Humidity</b></td> <td align="center"><b> Wind </b></td> <td align="center"><b>Ride</b></td> </tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td align="center"> 1 </td> <td align="center"> Sunny </td> <td align="center"> Hot </td> <td align="center"> High </td> <td align="center"> Weak </td> <td align="center"> 0 </td> </tr>
|
||||
<tr><td align="center"> 2 </td> <td align="center"> Sunny </td> <td align="center"> Hot </td> <td align="center"> High </td> <td align="center"> Strong </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 3 </td> <td align="center"> Overcast </td> <td align="center"> Hot </td> <td align="center"> High </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 4 </td> <td align="center"> Rain </td> <td align="center"> Mild </td> <td align="center"> High </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 5 </td> <td align="center"> Rain </td> <td align="center"> Cool </td> <td align="center"> Normal </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 6 </td> <td align="center"> Rain </td> <td align="center"> Cool </td> <td align="center"> Normal </td> <td align="center"> Strong </td> <td align="center"> 0 </td> </tr>
|
||||
<tr><td align="center"> 7 </td> <td align="center"> Overcast </td> <td align="center"> Cool </td> <td align="center"> Normal </td> <td align="center"> Strong </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 8 </td> <td align="center"> Sunny </td> <td align="center"> Mild </td> <td align="center"> High </td> <td align="center"> Weak </td> <td align="center"> 0 </td> </tr>
|
||||
<tr><td align="center"> 9 </td> <td align="center"> Sunny </td> <td align="center"> Cool </td> <td align="center"> Normal </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 10 </td> <td align="center"> Rain </td> <td align="center"> Mild </td> <td align="center"> Normal </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 11 </td> <td align="center"> Sunny </td> <td align="center"> Mild </td> <td align="center"> Normal </td> <td align="center"> Strong </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 12 </td> <td align="center"> Overcast </td> <td align="center"> Mild </td> <td align="center"> High </td> <td align="center"> Strong </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 13 </td> <td align="center"> Overcast </td> <td align="center"> Hot </td> <td align="center"> Normal </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 14 </td> <td align="center"> Rain </td> <td align="center"> Mild </td> <td align="center"> High </td> <td align="center"> Strong </td> <td align="center"> 0 </td> </tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div> <!-- col-xs-4 -->
|
||||
</div> <!-- cell row -->
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -207,7 +251,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._DecisionTrees-bs026.html">27</a></li>
|
||||
<li><a href="._DecisionTrees-bs027.html">28</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs019.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -176,51 +183,64 @@ MathJax.Hub.Config({
|
||||
<a name="part0019"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec18" class="anchor">Cancer Data again now with Decision Trees </h2>
|
||||
<h2 id="___sec18" class="anchor">Simple Python Code to read in Data </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Common imports</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler, OneHotEncoder
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.compose</span> <span style="color: #008000; font-weight: bold">import</span> ColumnTransformer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> Image
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">pydot</span> <span style="color: #008000; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Load the data</span>
|
||||
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
||||
<span style="color: #408080; font-style: italic"># Where to save the figures and data files</span>
|
||||
PROJECT_ROOT_DIR <span style="color: #666666">=</span> <span style="color: #BA2121">"Results"</span>
|
||||
FIGURE_ID <span style="color: #666666">=</span> <span style="color: #BA2121">"Results/FigureFiles"</span>
|
||||
DATA_ID <span style="color: #666666">=</span> <span style="color: #BA2121">"DataFiles/"</span>
|
||||
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(X_train<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(X_test<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
||||
logreg <span style="color: #666666">=</span> LogisticRegression(solver<span style="color: #666666">=</span><span style="color: #BA2121">'lbfgs'</span>)
|
||||
logreg<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test set accuracy with Logistic Regression: {:.2f}"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Support vector machine</span>
|
||||
svm <span style="color: #666666">=</span> SVC(gamma<span style="color: #666666">=</span><span style="color: #BA2121">'auto'</span>, C<span style="color: #666666">=100</span>)
|
||||
svm<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test set accuracy with SVM: {:.2f}"</span><span style="color: #666666">.</span>format(svm<span style="color: #666666">.</span>score(X_test,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Decision Trees</span>
|
||||
deep_tree_clf <span style="color: #666666">=</span> DecisionTreeClassifier(max_depth<span style="color: #666666">=</span><span style="color: #008000">None</span>)
|
||||
deep_tree_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test set accuracy with Decision Trees: {:.2f}"</span><span style="color: #666666">.</span>format(deep_tree_clf<span style="color: #666666">.</span>score(X_test,y_test)))
|
||||
<span style="color: #408080; font-style: italic">#now scale the data</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
scaler<span style="color: #666666">.</span>fit(X_train)
|
||||
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
||||
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
||||
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
||||
logreg<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test set accuracy Logistic Regression with scaled data: {:.2f}"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Support Vector Machine</span>
|
||||
svm<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test set accuracy SVM with scaled data: {:.2f}"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Decision Trees</span>
|
||||
deep_tree_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test set accuracy with Decision Trees and scaled data: {:.2f}"</span><span style="color: #666666">.</span>format(deep_tree_clf<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(PROJECT_ROOT_DIR):
|
||||
os<span style="color: #666666">.</span>mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(FIGURE_ID):
|
||||
os<span style="color: #666666">.</span>makedirs(FIGURE_ID)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(DATA_ID):
|
||||
os<span style="color: #666666">.</span>makedirs(DATA_ID)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">image_path</span>(fig_id):
|
||||
<span style="color: #008000; font-weight: bold">return</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(FIGURE_ID, fig_id)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">data_path</span>(dat_id):
|
||||
<span style="color: #008000; font-weight: bold">return</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(DATA_ID, dat_id)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">save_fig</span>(fig_id):
|
||||
plt<span style="color: #666666">.</span>savefig(image_path(fig_id) <span style="color: #666666">+</span> <span style="color: #BA2121">".png"</span>, format<span style="color: #666666">=</span><span style="color: #BA2121">'png'</span>)
|
||||
|
||||
infile <span style="color: #666666">=</span> <span style="color: #008000">open</span>(data_path(<span style="color: #BA2121">"ride.csv"</span>),<span style="color: #BA2121">'r'</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Read the experimental data with Pandas</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> display
|
||||
ridedata <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>read_csv(infile,names <span style="color: #666666">=</span> (<span style="color: #BA2121">'Outlook'</span>,<span style="color: #BA2121">'Temperature'</span>,<span style="color: #BA2121">'Humidity'</span>,<span style="color: #BA2121">'Wind'</span>,<span style="color: #BA2121">'Ride'</span>))
|
||||
ridedata <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>DataFrame(ridedata)
|
||||
display(ridedata)
|
||||
<span style="color: #408080; font-style: italic"># Features and targets</span>
|
||||
X <span style="color: #666666">=</span> ridedata<span style="color: #666666">.</span>loc[:, ridedata<span style="color: #666666">.</span>columns <span style="color: #666666">!=</span> <span style="color: #BA2121">'Ride'</span>]<span style="color: #666666">.</span>values
|
||||
display(X)
|
||||
y <span style="color: #666666">=</span> ridedata<span style="color: #666666">.</span>loc[:, ridedata<span style="color: #666666">.</span>columns <span style="color: #666666">==</span> <span style="color: #BA2121">'Ride'</span>]<span style="color: #666666">.</span>values
|
||||
display(y)
|
||||
<span style="color: #408080; font-style: italic"># Categorical variables to one-hot's</span>
|
||||
onehotencoder <span style="color: #666666">=</span> OneHotEncoder(categories<span style="color: #666666">=</span><span style="color: #BA2121">"auto"</span>)
|
||||
|
||||
X <span style="color: #666666">=</span> ColumnTransformer([(<span style="color: #BA2121">""</span>, onehotencoder)])<span style="color: #666666">.</span>fit_transform(X)
|
||||
y<span style="color: #666666">.</span>shape
|
||||
|
||||
display(X)
|
||||
display(y)
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
@@ -248,7 +268,7 @@ deep_tree_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<li><a href="._DecisionTrees-bs027.html">28</a></li>
|
||||
<li><a href="._DecisionTrees-bs028.html">29</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs020.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -176,74 +183,80 @@ MathJax.Hub.Config({
|
||||
<a name="part0020"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec19" class="anchor">Another example, the moons again </h2>
|
||||
<h2 id="___sec19" class="anchor">Computing the Gini Factor </h2>
|
||||
|
||||
<p>
|
||||
The above functions (gini, entropy and misclassification error) are
|
||||
important components of the so-called CART algorithm. We will discuss
|
||||
this algorithm below after we have discussed the information gain
|
||||
algorithm ID3.
|
||||
|
||||
<p>
|
||||
In the example here we have converted all our attributes into numerical values \( 0,1,2 \) etc.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">__future__</span> <span style="color: #008000; font-weight: bold">import</span> division, print_function, unicode_literals
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Split a dataset based on an attribute and an attribute value</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">test_split</span>(index, value, dataset):
|
||||
left, right <span style="color: #666666">=</span> <span style="color: #008000">list</span>(), <span style="color: #008000">list</span>()
|
||||
<span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> dataset:
|
||||
<span style="color: #008000; font-weight: bold">if</span> row[index] <span style="color: #666666"><</span> value:
|
||||
left<span style="color: #666666">.</span>append(row)
|
||||
<span style="color: #008000; font-weight: bold">else</span>:
|
||||
right<span style="color: #666666">.</span>append(row)
|
||||
<span style="color: #008000; font-weight: bold">return</span> left, right
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Calculate the Gini index for a split dataset</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">gini_index</span>(groups, classes):
|
||||
<span style="color: #408080; font-style: italic"># count all samples at split point</span>
|
||||
n_instances <span style="color: #666666">=</span> <span style="color: #008000">float</span>(<span style="color: #008000">sum</span>([<span style="color: #008000">len</span>(group) <span style="color: #008000; font-weight: bold">for</span> group <span style="color: #AA22FF; font-weight: bold">in</span> groups]))
|
||||
<span style="color: #408080; font-style: italic"># sum weighted Gini index for each group</span>
|
||||
gini <span style="color: #666666">=</span> <span style="color: #666666">0.0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> group <span style="color: #AA22FF; font-weight: bold">in</span> groups:
|
||||
size <span style="color: #666666">=</span> <span style="color: #008000">float</span>(<span style="color: #008000">len</span>(group))
|
||||
<span style="color: #408080; font-style: italic"># avoid divide by zero</span>
|
||||
<span style="color: #008000; font-weight: bold">if</span> size <span style="color: #666666">==</span> <span style="color: #666666">0</span>:
|
||||
<span style="color: #008000; font-weight: bold">continue</span>
|
||||
score <span style="color: #666666">=</span> <span style="color: #666666">0.0</span>
|
||||
<span style="color: #408080; font-style: italic"># score the group based on the score for each class</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> class_val <span style="color: #AA22FF; font-weight: bold">in</span> classes:
|
||||
p <span style="color: #666666">=</span> [row[<span style="color: #666666">-1</span>] <span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> group]<span style="color: #666666">.</span>count(class_val) <span style="color: #666666">/</span> size
|
||||
score <span style="color: #666666">+=</span> p <span style="color: #666666">*</span> p
|
||||
<span style="color: #408080; font-style: italic"># weight the group score by its relative size</span>
|
||||
gini <span style="color: #666666">+=</span> (<span style="color: #666666">1.0</span> <span style="color: #666666">-</span> score) <span style="color: #666666">*</span> (size <span style="color: #666666">/</span> n_instances)
|
||||
<span style="color: #008000; font-weight: bold">return</span> gini
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Common imports</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
||||
<span style="color: #408080; font-style: italic"># Select the best split point for a dataset</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">get_split</span>(dataset):
|
||||
class_values <span style="color: #666666">=</span> <span style="color: #008000">list</span>(<span style="color: #008000">set</span>(row[<span style="color: #666666">-1</span>] <span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> dataset))
|
||||
b_index, b_value, b_score, b_groups <span style="color: #666666">=</span> <span style="color: #666666">999</span>, <span style="color: #666666">999</span>, <span style="color: #666666">999</span>, <span style="color: #008000">None</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> index <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(dataset[<span style="color: #666666">0</span>])<span style="color: #666666">-1</span>):
|
||||
<span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> dataset:
|
||||
groups <span style="color: #666666">=</span> test_split(index, row[index], dataset)
|
||||
gini <span style="color: #666666">=</span> gini_index(groups, class_values)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'X</span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121"> < </span><span style="color: #BB6688; font-weight: bold">%.3f</span><span style="color: #BA2121"> Gini=</span><span style="color: #BB6688; font-weight: bold">%.3f</span><span style="color: #BA2121">'</span> <span style="color: #666666">%</span> ((index<span style="color: #666666">+1</span>), row[index], gini))
|
||||
<span style="color: #008000; font-weight: bold">if</span> gini <span style="color: #666666"><</span> b_score:
|
||||
b_index, b_value, b_score, b_groups <span style="color: #666666">=</span> index, row[index], gini, groups
|
||||
<span style="color: #008000; font-weight: bold">return</span> {<span style="color: #BA2121">'index'</span>:b_index, <span style="color: #BA2121">'value'</span>:b_value, <span style="color: #BA2121">'groups'</span>:b_groups}
|
||||
|
||||
dataset <span style="color: #666666">=</span> [[<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>],
|
||||
[<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">2</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">2</span>,<span style="color: #666666">2</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">2</span>,<span style="color: #666666">2</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>],
|
||||
[<span style="color: #666666">1</span>,<span style="color: #666666">2</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">0</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>],
|
||||
[<span style="color: #666666">0</span>,<span style="color: #666666">2</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">2</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">0</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">1</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">2</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>]]
|
||||
|
||||
<span style="color: #408080; font-style: italic"># to make this notebook's output stable across runs</span>
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">42</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># To plot pretty figures</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">matplotlib.colors</span> <span style="color: #008000; font-weight: bold">import</span> ListedColormap
|
||||
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">'axes.labelsize'</span>] <span style="color: #666666">=</span> <span style="color: #666666">14</span>
|
||||
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">'xtick.labelsize'</span>] <span style="color: #666666">=</span> <span style="color: #666666">12</span>
|
||||
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">'ytick.labelsize'</span>] <span style="color: #666666">=</span> <span style="color: #666666">12</span>
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_moons
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> export_graphviz
|
||||
|
||||
Xm, ym <span style="color: #666666">=</span> make_moons(n_samples<span style="color: #666666">=100</span>, noise<span style="color: #666666">=0.25</span>, random_state<span style="color: #666666">=53</span>)
|
||||
|
||||
deep_tree_clf1 <span style="color: #666666">=</span> DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>)
|
||||
deep_tree_clf2 <span style="color: #666666">=</span> DecisionTreeClassifier(min_samples_leaf<span style="color: #666666">=4</span>, random_state<span style="color: #666666">=42</span>)
|
||||
deep_tree_clf1<span style="color: #666666">.</span>fit(Xm, ym)
|
||||
deep_tree_clf2<span style="color: #666666">.</span>fit(Xm, ym)
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_decision_boundary</span>(clf, X, y, axes<span style="color: #666666">=</span>[<span style="color: #666666">0</span>, <span style="color: #666666">7.5</span>, <span style="color: #666666">0</span>, <span style="color: #666666">3</span>], iris<span style="color: #666666">=</span><span style="color: #008000">True</span>, legend<span style="color: #666666">=</span><span style="color: #008000">False</span>, plot_training<span style="color: #666666">=</span><span style="color: #008000">True</span>):
|
||||
x1s <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">0</span>], axes[<span style="color: #666666">1</span>], <span style="color: #666666">100</span>)
|
||||
x2s <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">2</span>], axes[<span style="color: #666666">3</span>], <span style="color: #666666">100</span>)
|
||||
x1, x2 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>meshgrid(x1s, x2s)
|
||||
X_new <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[x1<span style="color: #666666">.</span>ravel(), x2<span style="color: #666666">.</span>ravel()]
|
||||
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_new)<span style="color: #666666">.</span>reshape(x1<span style="color: #666666">.</span>shape)
|
||||
custom_cmap <span style="color: #666666">=</span> ListedColormap([<span style="color: #BA2121">'#fafab0'</span>,<span style="color: #BA2121">'#9898ff'</span>,<span style="color: #BA2121">'#a0faa0'</span>])
|
||||
plt<span style="color: #666666">.</span>contourf(x1, x2, y_pred, alpha<span style="color: #666666">=0.3</span>, cmap<span style="color: #666666">=</span>custom_cmap)
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> iris:
|
||||
custom_cmap2 <span style="color: #666666">=</span> ListedColormap([<span style="color: #BA2121">'#7d7d58'</span>,<span style="color: #BA2121">'#4c4c7f'</span>,<span style="color: #BA2121">'#507d50'</span>])
|
||||
plt<span style="color: #666666">.</span>contour(x1, x2, y_pred, cmap<span style="color: #666666">=</span>custom_cmap2, alpha<span style="color: #666666">=0.8</span>)
|
||||
<span style="color: #008000; font-weight: bold">if</span> plot_training:
|
||||
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==0</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==0</span>], <span style="color: #BA2121">"yo"</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Iris-Setosa"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==1</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==1</span>], <span style="color: #BA2121">"bs"</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Iris-Versicolor"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==2</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==2</span>], <span style="color: #BA2121">"g^"</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Iris-Virginica"</span>)
|
||||
plt<span style="color: #666666">.</span>axis(axes)
|
||||
<span style="color: #008000; font-weight: bold">if</span> iris:
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"Petal length"</span>, fontsize<span style="color: #666666">=14</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"Petal width"</span>, fontsize<span style="color: #666666">=14</span>)
|
||||
<span style="color: #008000; font-weight: bold">else</span>:
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r"$x_1$"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r"$x_2$"</span>, fontsize<span style="color: #666666">=18</span>, rotation<span style="color: #666666">=0</span>)
|
||||
<span style="color: #008000; font-weight: bold">if</span> legend:
|
||||
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">"lower right"</span>, fontsize<span style="color: #666666">=14</span>)
|
||||
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>, <span style="color: #666666">4</span>))
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
|
||||
plot_decision_boundary(deep_tree_clf1, Xm, ym, axes<span style="color: #666666">=</span>[<span style="color: #666666">-1.5</span>, <span style="color: #666666">2.5</span>, <span style="color: #666666">-1</span>, <span style="color: #666666">1.5</span>], iris<span style="color: #666666">=</span><span style="color: #008000">False</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"No restrictions"</span>, fontsize<span style="color: #666666">=16</span>)
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
|
||||
plot_decision_boundary(deep_tree_clf2, Xm, ym, axes<span style="color: #666666">=</span>[<span style="color: #666666">-1.5</span>, <span style="color: #666666">2.5</span>, <span style="color: #666666">-1</span>, <span style="color: #666666">1.5</span>], iris<span style="color: #666666">=</span><span style="color: #008000">False</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"min_samples_leaf = {}"</span><span style="color: #666666">.</span>format(deep_tree_clf2<span style="color: #666666">.</span>min_samples_leaf), fontsize<span style="color: #666666">=14</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
split <span style="color: #666666">=</span> get_split(dataset)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'Split: [X</span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121"> < </span><span style="color: #BB6688; font-weight: bold">%.3f</span><span style="color: #BA2121">]'</span> <span style="color: #666666">%</span> ((split[<span style="color: #BA2121">'index'</span>]<span style="color: #666666">+1</span>), split[<span style="color: #BA2121">'value'</span>]))
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
@@ -271,7 +284,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._DecisionTrees-bs028.html">29</a></li>
|
||||
<li><a href="._DecisionTrees-bs029.html">30</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs021.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -176,31 +183,39 @@ MathJax.Hub.Config({
|
||||
<a name="part0021"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec20" class="anchor">Playing around with regions </h2>
|
||||
<h2 id="___sec20" class="anchor">Entropy and the ID3 algorithm </h2>
|
||||
|
||||
<p>
|
||||
ID3, learns decision trees by constructing
|
||||
them topdown, beginning with the question <b>which attribute should be tested at the root of the tree</b>?
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">6</span>)
|
||||
Xs <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>, <span style="color: #666666">2</span>) <span style="color: #666666">-</span> <span style="color: #666666">0.5</span>
|
||||
ys <span style="color: #666666">=</span> (Xs[:, <span style="color: #666666">0</span>] <span style="color: #666666">></span> <span style="color: #666666">0</span>)<span style="color: #666666">.</span>astype(np<span style="color: #666666">.</span>float32) <span style="color: #666666">*</span> <span style="color: #666666">2</span>
|
||||
<ol>
|
||||
<li> Each instance attribute is evaluated using a statistical test to determine how well it alone classifies the training examples.</li>
|
||||
<li> The best attribute is selected and used as the test at the root node of the tree.</li>
|
||||
<li> A descendant of the root node is then created for each possible value of this attribute.</li>
|
||||
<li> Training examples are sorted to the appropriate descendant node.</li>
|
||||
<li> The entire process is then repeated using the training examples associated with each descendant node to select the best attribute to test at that point in the tree.</li>
|
||||
<li> This forms a greedy search for an acceptable decision tree, in which the algorithm never backtracks to reconsider earlier choices.</li>
|
||||
</ol>
|
||||
|
||||
angle <span style="color: #666666">=</span> np<span style="color: #666666">.</span>pi <span style="color: #666666">/</span> <span style="color: #666666">4</span>
|
||||
rotation_matrix <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[np<span style="color: #666666">.</span>cos(angle), <span style="color: #666666">-</span>np<span style="color: #666666">.</span>sin(angle)], [np<span style="color: #666666">.</span>sin(angle), np<span style="color: #666666">.</span>cos(angle)]])
|
||||
Xsr <span style="color: #666666">=</span> Xs<span style="color: #666666">.</span>dot(rotation_matrix)
|
||||
The ID3 algorithm selects, which attribute to test at each node in the
|
||||
tree.
|
||||
|
||||
tree_clf_s <span style="color: #666666">=</span> DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>)
|
||||
tree_clf_s<span style="color: #666666">.</span>fit(Xs, ys)
|
||||
tree_clf_sr <span style="color: #666666">=</span> DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>)
|
||||
tree_clf_sr<span style="color: #666666">.</span>fit(Xsr, ys)
|
||||
<p>
|
||||
We would like to select the attribute that is most useful for classifying
|
||||
examples.
|
||||
|
||||
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>, <span style="color: #666666">4</span>))
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
|
||||
plot_decision_boundary(tree_clf_s, Xs, ys, axes<span style="color: #666666">=</span>[<span style="color: #666666">-0.7</span>, <span style="color: #666666">0.7</span>, <span style="color: #666666">-0.7</span>, <span style="color: #666666">0.7</span>], iris<span style="color: #666666">=</span><span style="color: #008000">False</span>)
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
|
||||
plot_decision_boundary(tree_clf_sr, Xsr, ys, axes<span style="color: #666666">=</span>[<span style="color: #666666">-0.7</span>, <span style="color: #666666">0.7</span>, <span style="color: #666666">-0.7</span>, <span style="color: #666666">0.7</span>], iris<span style="color: #666666">=</span><span style="color: #008000">False</span>)
|
||||
<p>
|
||||
What is a good quantitative measure of the worth of an attribute?
|
||||
|
||||
<p>
|
||||
Information gain measures how well a given attribute separates the
|
||||
training examples according to their target classification.
|
||||
|
||||
<p>
|
||||
The ID3 algorithm uses this information gain measure to select among the candidate
|
||||
attributes at each step while growing the tree.
|
||||
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -227,7 +242,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._DecisionTrees-bs029.html">30</a></li>
|
||||
<li><a href="._DecisionTrees-bs030.html">31</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs022.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -176,25 +183,212 @@ MathJax.Hub.Config({
|
||||
<a name="part0022"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec21" class="anchor">Regression trees </h2>
|
||||
<h2 id="___sec21" class="anchor">Implementing the ID3 Algorithm </h2>
|
||||
|
||||
<p>
|
||||
import re
|
||||
import math
|
||||
from collections import deque
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Quadratic training set + noise</span>
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">42</span>)
|
||||
m <span style="color: #666666">=</span> <span style="color: #666666">200</span>
|
||||
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(m, <span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> <span style="color: #666666">4</span> <span style="color: #666666">*</span> (X <span style="color: #666666">-</span> <span style="color: #666666">0.5</span>) <span style="color: #666666">**</span> <span style="color: #666666">2</span>
|
||||
y <span style="color: #666666">=</span> y <span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(m, <span style="color: #666666">1</span>) <span style="color: #666666">/</span> <span style="color: #666666">10</span>
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- x is examples in training set -->
|
||||
<!-- y is set of attributes -->
|
||||
<!-- label is target attributes -->
|
||||
<!-- Node is a class which has properties values, childs, and next -->
|
||||
<!-- root is top node in the decision tree -->
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeRegressor
|
||||
<p>
|
||||
class Node(object):
|
||||
def __init__(self):
|
||||
self.value = None
|
||||
self.next = None
|
||||
self.childs = None
|
||||
|
||||
<p>
|
||||
<!-- Simple class of Decision Tree -->
|
||||
<!-- Aimed for who want to learn Decision Tree, so it is not optimized -->
|
||||
class DecisionTree(object):
|
||||
def __init__(self, sample, attributes, labels):
|
||||
self.sample = sample
|
||||
self.attributes = attributes
|
||||
self.labels = labels
|
||||
self.labelCodes = None
|
||||
self.labelCodesCount = None
|
||||
self.initLabelCodes()
|
||||
# print(self.labelCodes)
|
||||
self.root = None
|
||||
self.entropy = self.getEntropy([x for x in range(len(self.labels))])
|
||||
|
||||
<p>
|
||||
def initLabelCodes(self):
|
||||
self.labelCodes = []
|
||||
self.labelCodesCount = []
|
||||
for l in self.labels:
|
||||
if l not in self.labelCodes:
|
||||
self.labelCodes.append(l)
|
||||
self.labelCodesCount.append(0)
|
||||
self.labelCodesCount[self.labelCodes.index(l)] += 1
|
||||
|
||||
<p>
|
||||
def getLabelCodeId(self, sampleId):
|
||||
return self.labelCodes.index(self.labels[sampleId])
|
||||
|
||||
<p>
|
||||
def getAttributeValues(self, sampleIds, attributeId):
|
||||
vals = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in vals:
|
||||
vals.append(val)
|
||||
# print(vals)
|
||||
return vals
|
||||
|
||||
<p>
|
||||
def getEntropy(self, sampleIds):
|
||||
entropy = 0
|
||||
labelCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCount[self.getLabelCodeId(sid)] += 1
|
||||
# print("-ge", labelCount)
|
||||
for lv in labelCount:
|
||||
# print(lv)
|
||||
if lv != 0:
|
||||
entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
|
||||
else:
|
||||
entropy += 0
|
||||
return entropy
|
||||
|
||||
<p>
|
||||
def getDominantLabel(self, sampleIds):
|
||||
labelCodesCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
|
||||
return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
|
||||
|
||||
<p>
|
||||
def getInformationGain(self, sampleIds, attributeId):
|
||||
gain = self.getEntropy(sampleIds)
|
||||
attributeVals = []
|
||||
attributeValsCount = []
|
||||
attributeValsIds = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in attributeVals:
|
||||
attributeVals.append(val)
|
||||
attributeValsCount.append(0)
|
||||
attributeValsIds.append([])
|
||||
vid = attributeVals.index(val)
|
||||
attributeValsCount[vid] += 1
|
||||
attributeValsIds[vid].append(sid)
|
||||
# print("-gig", self.attributes[attributeId])
|
||||
for vc, vids in zip(attributeValsCount, attributeValsIds):
|
||||
# print("-gig", vids)
|
||||
gain -= vc/len(sampleIds) * self.getEntropy(vids)
|
||||
return gain
|
||||
|
||||
<p>
|
||||
def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
|
||||
attributesEntropy = [0] * len(attributeIds)
|
||||
for i, attId in zip(range(len(attributeIds)), attributeIds):
|
||||
attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
|
||||
maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
|
||||
return self.attributes[maxId], maxId
|
||||
|
||||
<p>
|
||||
def isSingleLabeled(self, sampleIds):
|
||||
label = self.labels[sampleIds[0]]
|
||||
for sid in sampleIds:
|
||||
if self.labels[sid] != label:
|
||||
return False
|
||||
return True
|
||||
|
||||
<p>
|
||||
def getLabel(self, sampleId):
|
||||
return self.labels[sampleId]
|
||||
|
||||
<p>
|
||||
def id3(self):
|
||||
sampleIds = [x for x in range(len(self.sample))]
|
||||
attributeIds = [x for x in range(len(self.attributes))]
|
||||
self.root = self.id3Recv(sampleIds, attributeIds, self.root)
|
||||
|
||||
<p>
|
||||
def id3Recv(self, sampleIds, attributeIds, root):
|
||||
root = Node() # Initialize current root
|
||||
if self.isSingleLabeled(sampleIds):
|
||||
root.value = self.labels[sampleIds[0]]
|
||||
return root
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) == 0:
|
||||
root.value = self.getDominantLabel(sampleIds)
|
||||
return root
|
||||
bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
|
||||
sampleIds, attributeIds)
|
||||
# print(bestAttrName)
|
||||
root.value = bestAttrName
|
||||
root.childs = [] # Create list of children
|
||||
for value in self.getAttributeValues(sampleIds, bestAttrId):
|
||||
# print(value)
|
||||
child = Node()
|
||||
child.value = value
|
||||
root.childs.append(child) # Append new child node to current
|
||||
# root
|
||||
childSampleIds = []
|
||||
for sid in sampleIds:
|
||||
if self.sample[sid][bestAttrId] == value:
|
||||
childSampleIds.append(sid)
|
||||
if len(childSampleIds) == 0:
|
||||
child.next = self.getDominantLabel(sampleIds)
|
||||
else:
|
||||
# print(bestAttrName, bestAttrId)
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) > 0 and bestAttrId in attributeIds:
|
||||
toRemove = attributeIds.index(bestAttrId)
|
||||
attributeIds.pop(toRemove)
|
||||
child.next = self.id3Recv(
|
||||
childSampleIds, attributeIds, child.next)
|
||||
return root
|
||||
|
||||
<p>
|
||||
def printTree(self):
|
||||
if self.root:
|
||||
roots = deque()
|
||||
roots.append(self.root)
|
||||
while len(roots) > 0:
|
||||
root = roots.popleft()
|
||||
print(root.value)
|
||||
if root.childs:
|
||||
for child in root.childs:
|
||||
print('({})'.format(child.value))
|
||||
roots.append(child.next)
|
||||
elif root.next:
|
||||
print(root.next)
|
||||
|
||||
<p>
|
||||
def test():
|
||||
f = open('DataFiles/rideclass.csv')
|
||||
attributes = f.readline().split(',')
|
||||
attributes = attributes[1:len(attributes)-1]
|
||||
print(attributes)
|
||||
sample = f.readlines()
|
||||
f.close()
|
||||
for i in range(len(sample)):
|
||||
sample[i] = re.sub('\d+,', '', sample[i])
|
||||
sample[i] = sample[i].strip().split(',')
|
||||
labels = []
|
||||
for s in sample:
|
||||
labels.append(s.pop())
|
||||
# print(sample)
|
||||
# print(labels)
|
||||
decisionTree = DecisionTree(sample, attributes, labels)
|
||||
print("System entropy {}".format(decisionTree.entropy))
|
||||
decisionTree.id3()
|
||||
decisionTree.printTree()
|
||||
|
||||
<p>
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
|
||||
tree_reg <span style="color: #666666">=</span> DecisionTreeRegressor(max_depth<span style="color: #666666">=2</span>, random_state<span style="color: #666666">=42</span>)
|
||||
tree_reg<span style="color: #666666">.</span>fit(X, y)
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -221,7 +415,7 @@ tree_reg<span style="color: #666666">.</span>fit(X, y)
|
||||
<li><a href="._DecisionTrees-bs030.html">31</a></li>
|
||||
<li><a href="._DecisionTrees-bs031.html">32</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs023.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -176,80 +183,51 @@ MathJax.Hub.Config({
|
||||
<a name="part0023"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec22" class="anchor">Final regressor code </h2>
|
||||
<h2 id="___sec22" class="anchor">Cancer Data again now with Decision Trees and other Methods </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeRegressor
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
|
||||
tree_reg1 <span style="color: #666666">=</span> DecisionTreeRegressor(random_state<span style="color: #666666">=42</span>, max_depth<span style="color: #666666">=2</span>)
|
||||
tree_reg2 <span style="color: #666666">=</span> DecisionTreeRegressor(random_state<span style="color: #666666">=42</span>, max_depth<span style="color: #666666">=3</span>)
|
||||
tree_reg1<span style="color: #666666">.</span>fit(X, y)
|
||||
tree_reg2<span style="color: #666666">.</span>fit(X, y)
|
||||
<span style="color: #408080; font-style: italic"># Load the data</span>
|
||||
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_regression_predictions</span>(tree_reg, X, y, axes<span style="color: #666666">=</span>[<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">-0.2</span>, <span style="color: #666666">1</span>], ylabel<span style="color: #666666">=</span><span style="color: #BA2121">"$y$"</span>):
|
||||
x1 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">0</span>], axes[<span style="color: #666666">1</span>], <span style="color: #666666">500</span>)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
y_pred <span style="color: #666666">=</span> tree_reg<span style="color: #666666">.</span>predict(x1)
|
||||
plt<span style="color: #666666">.</span>axis(axes)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"$x_1$"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
<span style="color: #008000; font-weight: bold">if</span> ylabel:
|
||||
plt<span style="color: #666666">.</span>ylabel(ylabel, fontsize<span style="color: #666666">=18</span>, rotation<span style="color: #666666">=0</span>)
|
||||
plt<span style="color: #666666">.</span>plot(X, y, <span style="color: #BA2121">"b."</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x1, y_pred, <span style="color: #BA2121">"r.-"</span>, linewidth<span style="color: #666666">=2</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">r"$\hat{y}$"</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>, <span style="color: #666666">4</span>))
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
|
||||
plot_regression_predictions(tree_reg1, X, y)
|
||||
<span style="color: #008000; font-weight: bold">for</span> split, style <span style="color: #AA22FF; font-weight: bold">in</span> ((<span style="color: #666666">0.1973</span>, <span style="color: #BA2121">"k-"</span>), (<span style="color: #666666">0.0917</span>, <span style="color: #BA2121">"k--"</span>), (<span style="color: #666666">0.7718</span>, <span style="color: #BA2121">"k--"</span>)):
|
||||
plt<span style="color: #666666">.</span>plot([split, split], [<span style="color: #666666">-0.2</span>, <span style="color: #666666">1</span>], style, linewidth<span style="color: #666666">=2</span>)
|
||||
plt<span style="color: #666666">.</span>text(<span style="color: #666666">0.21</span>, <span style="color: #666666">0.65</span>, <span style="color: #BA2121">"Depth=0"</span>, fontsize<span style="color: #666666">=15</span>)
|
||||
plt<span style="color: #666666">.</span>text(<span style="color: #666666">0.01</span>, <span style="color: #666666">0.2</span>, <span style="color: #BA2121">"Depth=1"</span>, fontsize<span style="color: #666666">=13</span>)
|
||||
plt<span style="color: #666666">.</span>text(<span style="color: #666666">0.65</span>, <span style="color: #666666">0.8</span>, <span style="color: #BA2121">"Depth=1"</span>, fontsize<span style="color: #666666">=13</span>)
|
||||
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">"upper center"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"max_depth=2"</span>, fontsize<span style="color: #666666">=14</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
|
||||
plot_regression_predictions(tree_reg2, X, y, ylabel<span style="color: #666666">=</span><span style="color: #008000">None</span>)
|
||||
<span style="color: #008000; font-weight: bold">for</span> split, style <span style="color: #AA22FF; font-weight: bold">in</span> ((<span style="color: #666666">0.1973</span>, <span style="color: #BA2121">"k-"</span>), (<span style="color: #666666">0.0917</span>, <span style="color: #BA2121">"k--"</span>), (<span style="color: #666666">0.7718</span>, <span style="color: #BA2121">"k--"</span>)):
|
||||
plt<span style="color: #666666">.</span>plot([split, split], [<span style="color: #666666">-0.2</span>, <span style="color: #666666">1</span>], style, linewidth<span style="color: #666666">=2</span>)
|
||||
<span style="color: #008000; font-weight: bold">for</span> split <span style="color: #AA22FF; font-weight: bold">in</span> (<span style="color: #666666">0.0458</span>, <span style="color: #666666">0.1298</span>, <span style="color: #666666">0.2873</span>, <span style="color: #666666">0.9040</span>):
|
||||
plt<span style="color: #666666">.</span>plot([split, split], [<span style="color: #666666">-0.2</span>, <span style="color: #666666">1</span>], <span style="color: #BA2121">"k:"</span>, linewidth<span style="color: #666666">=1</span>)
|
||||
plt<span style="color: #666666">.</span>text(<span style="color: #666666">0.3</span>, <span style="color: #666666">0.5</span>, <span style="color: #BA2121">"Depth=2"</span>, fontsize<span style="color: #666666">=13</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"max_depth=3"</span>, fontsize<span style="color: #666666">=14</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>tree_reg1 <span style="color: #666666">=</span> DecisionTreeRegressor(random_state<span style="color: #666666">=42</span>)
|
||||
tree_reg2 <span style="color: #666666">=</span> DecisionTreeRegressor(random_state<span style="color: #666666">=42</span>, min_samples_leaf<span style="color: #666666">=10</span>)
|
||||
tree_reg1<span style="color: #666666">.</span>fit(X, y)
|
||||
tree_reg2<span style="color: #666666">.</span>fit(X, y)
|
||||
|
||||
x1 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">500</span>)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
y_pred1 <span style="color: #666666">=</span> tree_reg1<span style="color: #666666">.</span>predict(x1)
|
||||
y_pred2 <span style="color: #666666">=</span> tree_reg2<span style="color: #666666">.</span>predict(x1)
|
||||
|
||||
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>, <span style="color: #666666">4</span>))
|
||||
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
|
||||
plt<span style="color: #666666">.</span>plot(X, y, <span style="color: #BA2121">"b."</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x1, y_pred1, <span style="color: #BA2121">"r.-"</span>, linewidth<span style="color: #666666">=2</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">r"$\hat{y}$"</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">-0.2</span>, <span style="color: #666666">1.1</span>])
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"$x_1$"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"$y$"</span>, fontsize<span style="color: #666666">=18</span>, rotation<span style="color: #666666">=0</span>)
|
||||
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">"upper center"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"No restrictions"</span>, fontsize<span style="color: #666666">=14</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
|
||||
plt<span style="color: #666666">.</span>plot(X, y, <span style="color: #BA2121">"b."</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x1, y_pred2, <span style="color: #BA2121">"r.-"</span>, linewidth<span style="color: #666666">=2</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">r"$\hat{y}$"</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">-0.2</span>, <span style="color: #666666">1.1</span>])
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"$x_1$"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"min_samples_leaf={}"</span><span style="color: #666666">.</span>format(tree_reg2<span style="color: #666666">.</span>min_samples_leaf), fontsize<span style="color: #666666">=14</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(X_train<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(X_test<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
||||
logreg <span style="color: #666666">=</span> LogisticRegression(solver<span style="color: #666666">=</span><span style="color: #BA2121">'lbfgs'</span>)
|
||||
logreg<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test set accuracy with Logistic Regression: {:.2f}"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Support vector machine</span>
|
||||
svm <span style="color: #666666">=</span> SVC(gamma<span style="color: #666666">=</span><span style="color: #BA2121">'auto'</span>, C<span style="color: #666666">=100</span>)
|
||||
svm<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test set accuracy with SVM: {:.2f}"</span><span style="color: #666666">.</span>format(svm<span style="color: #666666">.</span>score(X_test,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Decision Trees</span>
|
||||
deep_tree_clf <span style="color: #666666">=</span> DecisionTreeClassifier(max_depth<span style="color: #666666">=</span><span style="color: #008000">None</span>)
|
||||
deep_tree_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test set accuracy with Decision Trees: {:.2f}"</span><span style="color: #666666">.</span>format(deep_tree_clf<span style="color: #666666">.</span>score(X_test,y_test)))
|
||||
<span style="color: #408080; font-style: italic">#now scale the data</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
scaler<span style="color: #666666">.</span>fit(X_train)
|
||||
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
||||
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
||||
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
||||
logreg<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test set accuracy Logistic Regression with scaled data: {:.2f}"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Support Vector Machine</span>
|
||||
svm<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test set accuracy SVM with scaled data: {:.2f}"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Decision Trees</span>
|
||||
deep_tree_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test set accuracy with Decision Trees and scaled data: {:.2f}"</span><span style="color: #666666">.</span>format(deep_tree_clf<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
@@ -277,7 +255,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._DecisionTrees-bs031.html">32</a></li>
|
||||
<li><a href="._DecisionTrees-bs032.html">33</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs024.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -176,18 +183,76 @@ MathJax.Hub.Config({
|
||||
<a name="part0024"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec23" class="anchor">Pros and cons of trees, pros </h2>
|
||||
<h2 id="___sec23" class="anchor">Another example, the moons again </h2>
|
||||
<p>
|
||||
|
||||
<ul>
|
||||
<li> White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines)</li>
|
||||
<li> Trees are very easy to explain to people. In fact, they are even easier to explain than linear regression!</li>
|
||||
<li> No feature normalization needed</li>
|
||||
<li> Tree models can handle both continuous and categorical data (Classification and Regression Trees)</li>
|
||||
<li> Can model nonlinear relationships</li>
|
||||
<li> Can model interactions between the different descriptive features</li>
|
||||
<li> Trees can be displayed graphically, and are easily interpreted even by a non-expert (especially if they are small)</li>
|
||||
</ul>
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">__future__</span> <span style="color: #008000; font-weight: bold">import</span> division, print_function, unicode_literals
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Common imports</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># to make this notebook's output stable across runs</span>
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">42</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># To plot pretty figures</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">matplotlib.colors</span> <span style="color: #008000; font-weight: bold">import</span> ListedColormap
|
||||
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">'axes.labelsize'</span>] <span style="color: #666666">=</span> <span style="color: #666666">14</span>
|
||||
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">'xtick.labelsize'</span>] <span style="color: #666666">=</span> <span style="color: #666666">12</span>
|
||||
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">'ytick.labelsize'</span>] <span style="color: #666666">=</span> <span style="color: #666666">12</span>
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_moons
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> export_graphviz
|
||||
|
||||
Xm, ym <span style="color: #666666">=</span> make_moons(n_samples<span style="color: #666666">=100</span>, noise<span style="color: #666666">=0.25</span>, random_state<span style="color: #666666">=53</span>)
|
||||
|
||||
deep_tree_clf1 <span style="color: #666666">=</span> DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>)
|
||||
deep_tree_clf2 <span style="color: #666666">=</span> DecisionTreeClassifier(min_samples_leaf<span style="color: #666666">=4</span>, random_state<span style="color: #666666">=42</span>)
|
||||
deep_tree_clf1<span style="color: #666666">.</span>fit(Xm, ym)
|
||||
deep_tree_clf2<span style="color: #666666">.</span>fit(Xm, ym)
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_decision_boundary</span>(clf, X, y, axes<span style="color: #666666">=</span>[<span style="color: #666666">0</span>, <span style="color: #666666">7.5</span>, <span style="color: #666666">0</span>, <span style="color: #666666">3</span>], iris<span style="color: #666666">=</span><span style="color: #008000">True</span>, legend<span style="color: #666666">=</span><span style="color: #008000">False</span>, plot_training<span style="color: #666666">=</span><span style="color: #008000">True</span>):
|
||||
x1s <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">0</span>], axes[<span style="color: #666666">1</span>], <span style="color: #666666">100</span>)
|
||||
x2s <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">2</span>], axes[<span style="color: #666666">3</span>], <span style="color: #666666">100</span>)
|
||||
x1, x2 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>meshgrid(x1s, x2s)
|
||||
X_new <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[x1<span style="color: #666666">.</span>ravel(), x2<span style="color: #666666">.</span>ravel()]
|
||||
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_new)<span style="color: #666666">.</span>reshape(x1<span style="color: #666666">.</span>shape)
|
||||
custom_cmap <span style="color: #666666">=</span> ListedColormap([<span style="color: #BA2121">'#fafab0'</span>,<span style="color: #BA2121">'#9898ff'</span>,<span style="color: #BA2121">'#a0faa0'</span>])
|
||||
plt<span style="color: #666666">.</span>contourf(x1, x2, y_pred, alpha<span style="color: #666666">=0.3</span>, cmap<span style="color: #666666">=</span>custom_cmap)
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> iris:
|
||||
custom_cmap2 <span style="color: #666666">=</span> ListedColormap([<span style="color: #BA2121">'#7d7d58'</span>,<span style="color: #BA2121">'#4c4c7f'</span>,<span style="color: #BA2121">'#507d50'</span>])
|
||||
plt<span style="color: #666666">.</span>contour(x1, x2, y_pred, cmap<span style="color: #666666">=</span>custom_cmap2, alpha<span style="color: #666666">=0.8</span>)
|
||||
<span style="color: #008000; font-weight: bold">if</span> plot_training:
|
||||
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==0</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==0</span>], <span style="color: #BA2121">"yo"</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Iris-Setosa"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==1</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==1</span>], <span style="color: #BA2121">"bs"</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Iris-Versicolor"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==2</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==2</span>], <span style="color: #BA2121">"g^"</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Iris-Virginica"</span>)
|
||||
plt<span style="color: #666666">.</span>axis(axes)
|
||||
<span style="color: #008000; font-weight: bold">if</span> iris:
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"Petal length"</span>, fontsize<span style="color: #666666">=14</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"Petal width"</span>, fontsize<span style="color: #666666">=14</span>)
|
||||
<span style="color: #008000; font-weight: bold">else</span>:
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r"$x_1$"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r"$x_2$"</span>, fontsize<span style="color: #666666">=18</span>, rotation<span style="color: #666666">=0</span>)
|
||||
<span style="color: #008000; font-weight: bold">if</span> legend:
|
||||
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">"lower right"</span>, fontsize<span style="color: #666666">=14</span>)
|
||||
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>, <span style="color: #666666">4</span>))
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
|
||||
plot_decision_boundary(deep_tree_clf1, Xm, ym, axes<span style="color: #666666">=</span>[<span style="color: #666666">-1.5</span>, <span style="color: #666666">2.5</span>, <span style="color: #666666">-1</span>, <span style="color: #666666">1.5</span>], iris<span style="color: #666666">=</span><span style="color: #008000">False</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"No restrictions"</span>, fontsize<span style="color: #666666">=16</span>)
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
|
||||
plot_decision_boundary(deep_tree_clf2, Xm, ym, axes<span style="color: #666666">=</span>[<span style="color: #666666">-1.5</span>, <span style="color: #666666">2.5</span>, <span style="color: #666666">-1</span>, <span style="color: #666666">1.5</span>], iris<span style="color: #666666">=</span><span style="color: #008000">False</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"min_samples_leaf = {}"</span><span style="color: #666666">.</span>format(deep_tree_clf2<span style="color: #666666">.</span>min_samples_leaf), fontsize<span style="color: #666666">=14</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
@@ -212,6 +277,8 @@ MathJax.Hub.Config({
|
||||
<li><a href="._DecisionTrees-bs031.html">32</a></li>
|
||||
<li><a href="._DecisionTrees-bs032.html">33</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs025.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -176,20 +183,31 @@ MathJax.Hub.Config({
|
||||
<a name="part0025"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec24" class="anchor">Disadvantages </h2>
|
||||
<h2 id="___sec24" class="anchor">Playing around with regions </h2>
|
||||
<p>
|
||||
|
||||
<ul>
|
||||
<li> Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches</li>
|
||||
<li> If continuous features are used the tree may become quite large and hence less interpretable</li>
|
||||
<li> Decision trees are prone to overfit the training data and hence do not well generalize the data if no stopping criteria or improvements like pruning, boosting or bagging are implemented</li>
|
||||
<li> Small changes in the data may lead to a completely different tree. This issue can be addressed by using ensemble methods like bagging, boosting or random forests</li>
|
||||
<li> Unbalanced datasets where some target feature values occur much more frequently than others may lead to biased trees since the frequently occurring feature values are preferred over the less frequently occurring ones.</li>
|
||||
<li> If the number of features is relatively large (high dimensional) and the number of instances is relatively low, the tree might overfit the data</li>
|
||||
<li> Features with many levels may be preferred over features with less levels since for them it is <em>more easy</em> to split the dataset such that the sub datasets only contain pure target feature values. This issue can be addressed by preferring for instance the information gain ratio as splitting criteria over information gain</li>
|
||||
</ul>
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">6</span>)
|
||||
Xs <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>, <span style="color: #666666">2</span>) <span style="color: #666666">-</span> <span style="color: #666666">0.5</span>
|
||||
ys <span style="color: #666666">=</span> (Xs[:, <span style="color: #666666">0</span>] <span style="color: #666666">></span> <span style="color: #666666">0</span>)<span style="color: #666666">.</span>astype(np<span style="color: #666666">.</span>float32) <span style="color: #666666">*</span> <span style="color: #666666">2</span>
|
||||
|
||||
However, by aggregating many decision trees, using methods like bagging, random forests, and boosting, the predictive performance of trees can be substantially improved.
|
||||
angle <span style="color: #666666">=</span> np<span style="color: #666666">.</span>pi<span style="color: #666666">/4</span>
|
||||
rotation_matrix <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[np<span style="color: #666666">.</span>cos(angle), <span style="color: #666666">-</span>np<span style="color: #666666">.</span>sin(angle)], [np<span style="color: #666666">.</span>sin(angle), np<span style="color: #666666">.</span>cos(angle)]])
|
||||
Xsr <span style="color: #666666">=</span> Xs<span style="color: #666666">.</span>dot(rotation_matrix)
|
||||
|
||||
tree_clf_s <span style="color: #666666">=</span> DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>)
|
||||
tree_clf_s<span style="color: #666666">.</span>fit(Xs, ys)
|
||||
tree_clf_sr <span style="color: #666666">=</span> DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>)
|
||||
tree_clf_sr<span style="color: #666666">.</span>fit(Xsr, ys)
|
||||
|
||||
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>, <span style="color: #666666">4</span>))
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
|
||||
plot_decision_boundary(tree_clf_s, Xs, ys, axes<span style="color: #666666">=</span>[<span style="color: #666666">-0.7</span>, <span style="color: #666666">0.7</span>, <span style="color: #666666">-0.7</span>, <span style="color: #666666">0.7</span>], iris<span style="color: #666666">=</span><span style="color: #008000">False</span>)
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
|
||||
plot_decision_boundary(tree_clf_sr, Xsr, ys, axes<span style="color: #666666">=</span>[<span style="color: #666666">-0.7</span>, <span style="color: #666666">0.7</span>, <span style="color: #666666">-0.7</span>, <span style="color: #666666">0.7</span>], iris<span style="color: #666666">=</span><span style="color: #008000">False</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -214,6 +232,9 @@ However, by aggregating many decision trees, using methods like bagging, random
|
||||
<li><a href="._DecisionTrees-bs031.html">32</a></li>
|
||||
<li><a href="._DecisionTrees-bs032.html">33</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs034.html">35</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs026.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -176,46 +183,25 @@ MathJax.Hub.Config({
|
||||
<a name="part0026"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec25" class="anchor">Bagging </h2>
|
||||
|
||||
<h2 id="___sec25" class="anchor">Regression trees </h2>
|
||||
<p>
|
||||
The <b>plain</b> decision trees suffer from high
|
||||
variance. This means that if we split the training data into two parts
|
||||
at random, and fit a decision tree to both halves, the results that we
|
||||
get could be quite different. In contrast, a procedure with low
|
||||
variance will yield similar results if applied repeatedly to distinct
|
||||
data sets; linear regression tends to have low variance, if the ratio
|
||||
of \( n \) to \( p \) is moderately large.
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Quadratic training set + noise</span>
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">42</span>)
|
||||
m <span style="color: #666666">=</span> <span style="color: #666666">200</span>
|
||||
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(m, <span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> <span style="color: #666666">4</span> <span style="color: #666666">*</span> (X <span style="color: #666666">-</span> <span style="color: #666666">0.5</span>) <span style="color: #666666">**</span> <span style="color: #666666">2</span>
|
||||
y <span style="color: #666666">=</span> y <span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(m, <span style="color: #666666">1</span>) <span style="color: #666666">/</span> <span style="color: #666666">10</span>
|
||||
</pre></div>
|
||||
<p>
|
||||
<b>Bootstrap aggregation</b>, or just <b>bagging</b>, is a
|
||||
general-purpose procedure for reducing the variance of a statistical
|
||||
learning method.
|
||||
|
||||
<p>
|
||||
Bagging typically results in improved accuracy
|
||||
over prediction using a single tree. Unfortunately, however, it can be
|
||||
difficult to interpret the resulting model. Recall that one of the
|
||||
advantages of decision trees is the attractive and easily interpreted
|
||||
diagram that results.
|
||||
|
||||
<p>
|
||||
However, when we bag a large number of trees, it is no longer
|
||||
possible to represent the resulting statistical learning procedure
|
||||
using a single tree, and it is no longer clear which variables are
|
||||
most important to the procedure. Thus, bagging improves prediction
|
||||
accuracy at the expense of interpretability. Although the collection
|
||||
of bagged trees is much more difficult to interpret than a single
|
||||
tree, one can obtain an overall summary of the importance of each
|
||||
predictor using the MSE (for bagging regression trees) or the Gini
|
||||
index (for bagging classification trees). In the case of bagging
|
||||
regression trees, we can record the total amount that the MSE is
|
||||
decreased due to splits over a given predictor, averaged over all \( B \) possible
|
||||
trees. A large value indicates an important predictor. Similarly, in
|
||||
the context of bagging classification trees, we can add up the total
|
||||
amount that the Gini index is decreased by splits over a given
|
||||
predictor, averaged over all \( B \) trees.
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeRegressor
|
||||
|
||||
tree_reg <span style="color: #666666">=</span> DecisionTreeRegressor(max_depth<span style="color: #666666">=2</span>, random_state<span style="color: #666666">=42</span>)
|
||||
tree_reg<span style="color: #666666">.</span>fit(X, y)
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -239,6 +225,10 @@ predictor, averaged over all \( B \) trees.
|
||||
<li><a href="._DecisionTrees-bs031.html">32</a></li>
|
||||
<li><a href="._DecisionTrees-bs032.html">33</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs034.html">35</a></li>
|
||||
<li><a href="._DecisionTrees-bs035.html">36</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs027.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -176,21 +183,79 @@ MathJax.Hub.Config({
|
||||
<a name="part0027"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec26" class="anchor">Simple example, head or tail </h2>
|
||||
<h2 id="___sec26" class="anchor">Final regressor code </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>heads_proba <span style="color: #666666">=</span> <span style="color: #666666">0.51</span>
|
||||
coin_tosses <span style="color: #666666">=</span> (np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">10000</span>, <span style="color: #666666">10</span>) <span style="color: #666666"><</span> heads_proba)<span style="color: #666666">.</span>astype(np<span style="color: #666666">.</span>int32)
|
||||
cumulative_heads_ratio <span style="color: #666666">=</span> np<span style="color: #666666">.</span>cumsum(coin_tosses, axis<span style="color: #666666">=0</span>) <span style="color: #666666">/</span> np<span style="color: #666666">.</span>arange(<span style="color: #666666">1</span>, <span style="color: #666666">10001</span>)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">8</span>,<span style="color: #666666">3.5</span>))
|
||||
plt<span style="color: #666666">.</span>plot(cumulative_heads_ratio)
|
||||
plt<span style="color: #666666">.</span>plot([<span style="color: #666666">0</span>, <span style="color: #666666">10000</span>], [<span style="color: #666666">0.51</span>, <span style="color: #666666">0.51</span>], <span style="color: #BA2121">"k--"</span>, linewidth<span style="color: #666666">=2</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"51%"</span>)
|
||||
plt<span style="color: #666666">.</span>plot([<span style="color: #666666">0</span>, <span style="color: #666666">10000</span>], [<span style="color: #666666">0.5</span>, <span style="color: #666666">0.5</span>], <span style="color: #BA2121">"k-"</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"50%"</span>)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"Number of coin tosses"</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"Heads ratio"</span>)
|
||||
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">"lower right"</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>, <span style="color: #666666">10000</span>, <span style="color: #666666">0.42</span>, <span style="color: #666666">0.58</span>])
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeRegressor
|
||||
|
||||
tree_reg1 <span style="color: #666666">=</span> DecisionTreeRegressor(random_state<span style="color: #666666">=42</span>, max_depth<span style="color: #666666">=2</span>)
|
||||
tree_reg2 <span style="color: #666666">=</span> DecisionTreeRegressor(random_state<span style="color: #666666">=42</span>, max_depth<span style="color: #666666">=3</span>)
|
||||
tree_reg1<span style="color: #666666">.</span>fit(X, y)
|
||||
tree_reg2<span style="color: #666666">.</span>fit(X, y)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_regression_predictions</span>(tree_reg, X, y, axes<span style="color: #666666">=</span>[<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">-0.2</span>, <span style="color: #666666">1</span>], ylabel<span style="color: #666666">=</span><span style="color: #BA2121">"$y$"</span>):
|
||||
x1 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">0</span>], axes[<span style="color: #666666">1</span>], <span style="color: #666666">500</span>)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
y_pred <span style="color: #666666">=</span> tree_reg<span style="color: #666666">.</span>predict(x1)
|
||||
plt<span style="color: #666666">.</span>axis(axes)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"$x_1$"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
<span style="color: #008000; font-weight: bold">if</span> ylabel:
|
||||
plt<span style="color: #666666">.</span>ylabel(ylabel, fontsize<span style="color: #666666">=18</span>, rotation<span style="color: #666666">=0</span>)
|
||||
plt<span style="color: #666666">.</span>plot(X, y, <span style="color: #BA2121">"b."</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x1, y_pred, <span style="color: #BA2121">"r.-"</span>, linewidth<span style="color: #666666">=2</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">r"$\hat{y}$"</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>, <span style="color: #666666">4</span>))
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
|
||||
plot_regression_predictions(tree_reg1, X, y)
|
||||
<span style="color: #008000; font-weight: bold">for</span> split, style <span style="color: #AA22FF; font-weight: bold">in</span> ((<span style="color: #666666">0.1973</span>, <span style="color: #BA2121">"k-"</span>), (<span style="color: #666666">0.0917</span>, <span style="color: #BA2121">"k--"</span>), (<span style="color: #666666">0.7718</span>, <span style="color: #BA2121">"k--"</span>)):
|
||||
plt<span style="color: #666666">.</span>plot([split, split], [<span style="color: #666666">-0.2</span>, <span style="color: #666666">1</span>], style, linewidth<span style="color: #666666">=2</span>)
|
||||
plt<span style="color: #666666">.</span>text(<span style="color: #666666">0.21</span>, <span style="color: #666666">0.65</span>, <span style="color: #BA2121">"Depth=0"</span>, fontsize<span style="color: #666666">=15</span>)
|
||||
plt<span style="color: #666666">.</span>text(<span style="color: #666666">0.01</span>, <span style="color: #666666">0.2</span>, <span style="color: #BA2121">"Depth=1"</span>, fontsize<span style="color: #666666">=13</span>)
|
||||
plt<span style="color: #666666">.</span>text(<span style="color: #666666">0.65</span>, <span style="color: #666666">0.8</span>, <span style="color: #BA2121">"Depth=1"</span>, fontsize<span style="color: #666666">=13</span>)
|
||||
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">"upper center"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"max_depth=2"</span>, fontsize<span style="color: #666666">=14</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
|
||||
plot_regression_predictions(tree_reg2, X, y, ylabel<span style="color: #666666">=</span><span style="color: #008000">None</span>)
|
||||
<span style="color: #008000; font-weight: bold">for</span> split, style <span style="color: #AA22FF; font-weight: bold">in</span> ((<span style="color: #666666">0.1973</span>, <span style="color: #BA2121">"k-"</span>), (<span style="color: #666666">0.0917</span>, <span style="color: #BA2121">"k--"</span>), (<span style="color: #666666">0.7718</span>, <span style="color: #BA2121">"k--"</span>)):
|
||||
plt<span style="color: #666666">.</span>plot([split, split], [<span style="color: #666666">-0.2</span>, <span style="color: #666666">1</span>], style, linewidth<span style="color: #666666">=2</span>)
|
||||
<span style="color: #008000; font-weight: bold">for</span> split <span style="color: #AA22FF; font-weight: bold">in</span> (<span style="color: #666666">0.0458</span>, <span style="color: #666666">0.1298</span>, <span style="color: #666666">0.2873</span>, <span style="color: #666666">0.9040</span>):
|
||||
plt<span style="color: #666666">.</span>plot([split, split], [<span style="color: #666666">-0.2</span>, <span style="color: #666666">1</span>], <span style="color: #BA2121">"k:"</span>, linewidth<span style="color: #666666">=1</span>)
|
||||
plt<span style="color: #666666">.</span>text(<span style="color: #666666">0.3</span>, <span style="color: #666666">0.5</span>, <span style="color: #BA2121">"Depth=2"</span>, fontsize<span style="color: #666666">=13</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"max_depth=3"</span>, fontsize<span style="color: #666666">=14</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>tree_reg1 <span style="color: #666666">=</span> DecisionTreeRegressor(random_state<span style="color: #666666">=42</span>)
|
||||
tree_reg2 <span style="color: #666666">=</span> DecisionTreeRegressor(random_state<span style="color: #666666">=42</span>, min_samples_leaf<span style="color: #666666">=10</span>)
|
||||
tree_reg1<span style="color: #666666">.</span>fit(X, y)
|
||||
tree_reg2<span style="color: #666666">.</span>fit(X, y)
|
||||
|
||||
x1 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">500</span>)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
y_pred1 <span style="color: #666666">=</span> tree_reg1<span style="color: #666666">.</span>predict(x1)
|
||||
y_pred2 <span style="color: #666666">=</span> tree_reg2<span style="color: #666666">.</span>predict(x1)
|
||||
|
||||
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>, <span style="color: #666666">4</span>))
|
||||
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
|
||||
plt<span style="color: #666666">.</span>plot(X, y, <span style="color: #BA2121">"b."</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x1, y_pred1, <span style="color: #BA2121">"r.-"</span>, linewidth<span style="color: #666666">=2</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">r"$\hat{y}$"</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">-0.2</span>, <span style="color: #666666">1.1</span>])
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"$x_1$"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"$y$"</span>, fontsize<span style="color: #666666">=18</span>, rotation<span style="color: #666666">=0</span>)
|
||||
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">"upper center"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"No restrictions"</span>, fontsize<span style="color: #666666">=14</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
|
||||
plt<span style="color: #666666">.</span>plot(X, y, <span style="color: #BA2121">"b."</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x1, y_pred2, <span style="color: #BA2121">"r.-"</span>, linewidth<span style="color: #666666">=2</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">r"$\hat{y}$"</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">-0.2</span>, <span style="color: #666666">1.1</span>])
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"$x_1$"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"min_samples_leaf={}"</span><span style="color: #666666">.</span>format(tree_reg2<span style="color: #666666">.</span>min_samples_leaf), fontsize<span style="color: #666666">=14</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
@@ -215,6 +280,11 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._DecisionTrees-bs031.html">32</a></li>
|
||||
<li><a href="._DecisionTrees-bs032.html">33</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs034.html">35</a></li>
|
||||
<li><a href="._DecisionTrees-bs035.html">36</a></li>
|
||||
<li><a href="._DecisionTrees-bs036.html">37</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs028.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -176,48 +183,18 @@ MathJax.Hub.Config({
|
||||
<a name="part0028"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec27" class="anchor">Random forests </h2>
|
||||
<h2 id="___sec27" class="anchor">Pros and cons of trees, pros </h2>
|
||||
|
||||
<p>
|
||||
Random forests provide an improvement over bagged trees by way of a
|
||||
small tweak that decorrelates the trees.
|
||||
<ul>
|
||||
<li> White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines)</li>
|
||||
<li> Trees are very easy to explain to people. In fact, they are even easier to explain than linear regression!</li>
|
||||
<li> No feature normalization needed</li>
|
||||
<li> Tree models can handle both continuous and categorical data (Classification and Regression Trees)</li>
|
||||
<li> Can model nonlinear relationships</li>
|
||||
<li> Can model interactions between the different descriptive features</li>
|
||||
<li> Trees can be displayed graphically, and are easily interpreted even by a non-expert (especially if they are small)</li>
|
||||
</ul>
|
||||
|
||||
<p>
|
||||
As in bagging, we build a
|
||||
number of decision trees on bootstrapped training samples. But when
|
||||
building these decision trees, each time a split in a tree is
|
||||
considered, a random sample of \( m \) predictors is chosen as split
|
||||
candidates from the full set of \( p \) predictors. The split is allowed to
|
||||
use only one of those \( m \) predictors.
|
||||
|
||||
<p>
|
||||
A fresh sample of \( m \) predictors is
|
||||
taken at each split, and typically we choose
|
||||
|
||||
$$
|
||||
m\approx \sqrt{p}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
In building a random forest, at
|
||||
each split in the tree, the algorithm is not even allowed to consider
|
||||
a majority of the available predictors.
|
||||
|
||||
<p>
|
||||
The reason for this is rather clever. Suppose that there is one very
|
||||
strong predictor in the data set, along with a number of other
|
||||
moderately strong predictors. Then in the collection of bagged
|
||||
variable importance random forest trees, most or all of the trees will
|
||||
use this strong predictor in the top split. Consequently, all of the
|
||||
bagged trees will look quite similar to each other. Hence the
|
||||
predictions from the bagged trees will be highly correlated.
|
||||
Unfortunately, averaging many highly correlated quantities does not
|
||||
lead to as large of a reduction in variance as averaging many
|
||||
uncorrelated quanti- ties. In particular, this means that bagging will
|
||||
not lead to a substantial reduction in variance over a single tree in
|
||||
this setting.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
@@ -238,6 +215,12 @@ this setting.
|
||||
<li><a href="._DecisionTrees-bs031.html">32</a></li>
|
||||
<li><a href="._DecisionTrees-bs032.html">33</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs034.html">35</a></li>
|
||||
<li><a href="._DecisionTrees-bs035.html">36</a></li>
|
||||
<li><a href="._DecisionTrees-bs036.html">37</a></li>
|
||||
<li><a href="._DecisionTrees-bs037.html">38</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs029.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -176,21 +183,20 @@ MathJax.Hub.Config({
|
||||
<a name="part0029"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec28" class="anchor">A simple scikit-learn example </h2>
|
||||
<p>
|
||||
<h2 id="___sec28" class="anchor">Disadvantages </h2>
|
||||
|
||||
<ul>
|
||||
<li> Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches</li>
|
||||
<li> If continuous features are used the tree may become quite large and hence less interpretable</li>
|
||||
<li> Decision trees are prone to overfit the training data and hence do not well generalize the data if no stopping criteria or improvements like pruning, boosting or bagging are implemented</li>
|
||||
<li> Small changes in the data may lead to a completely different tree. This issue can be addressed by using ensemble methods like bagging, boosting or random forests</li>
|
||||
<li> Unbalanced datasets where some target feature values occur much more frequently than others may lead to biased trees since the frequently occurring feature values are preferred over the less frequently occurring ones.</li>
|
||||
<li> If the number of features is relatively large (high dimensional) and the number of instances is relatively low, the tree might overfit the data</li>
|
||||
<li> Features with many levels may be preferred over features with less levels since for them it is <em>more easy</em> to split the dataset such that the sub datasets only contain pure target feature values. This issue can be addressed by preferring for instance the information gain ratio as splitting criteria over information gain</li>
|
||||
</ul>
|
||||
|
||||
However, by aggregating many decision trees, using methods like bagging, random forests, and boosting, the predictive performance of trees can be substantially improved.
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> LabelEncoder
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
|
||||
<span style="color: #408080; font-style: italic"># Data set not specificied</span>
|
||||
X <span style="color: #666666">=</span> dataset<span style="color: #666666">.</span>XXX
|
||||
Y <span style="color: #666666">=</span> dataset<span style="color: #666666">.</span>YYY
|
||||
<span style="color: #408080; font-style: italic">#Instantiate the model with 100 trees and entropy as splitting criteria</span>
|
||||
Random_Forest_model <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=100</span>,criterion<span style="color: #666666">=</span><span style="color: #BA2121">"entropy"</span>)
|
||||
<span style="color: #408080; font-style: italic">#Cross validation</span>
|
||||
accuracy <span style="color: #666666">=</span> cross_validate(Random_Forest_model,X,Y,cv<span style="color: #666666">=10</span>)[<span style="color: #BA2121">'test_score'</span>]
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -211,6 +217,11 @@ accuracy <span style="color: #666666">=</span> cross_validate(Random_Forest_mode
|
||||
<li><a href="._DecisionTrees-bs031.html">32</a></li>
|
||||
<li><a href="._DecisionTrees-bs032.html">33</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs034.html">35</a></li>
|
||||
<li><a href="._DecisionTrees-bs035.html">36</a></li>
|
||||
<li><a href="._DecisionTrees-bs036.html">37</a></li>
|
||||
<li><a href="._DecisionTrees-bs037.html">38</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs030.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -176,61 +183,22 @@ MathJax.Hub.Config({
|
||||
<a name="part0030"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec29" class="anchor">Please, not the moons again! </h2>
|
||||
<h2 id="___sec29" class="anchor">Bagging </h2>
|
||||
|
||||
<p>
|
||||
The <b>plain</b> decision trees suffer from high
|
||||
variance. This means that if we split the training data into two parts
|
||||
at random, and fit a decision tree to both halves, the results that we
|
||||
get could be quite different. In contrast, a procedure with low
|
||||
variance will yield similar results if applied repeatedly to distinct
|
||||
data sets; linear regression tends to have low variance, if the ratio
|
||||
of \( n \) to \( p \) is moderately large.
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_moons
|
||||
|
||||
X, y <span style="color: #666666">=</span> make_moons(n_samples<span style="color: #666666">=500</span>, noise<span style="color: #666666">=0.30</span>, random_state<span style="color: #666666">=42</span>)
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(X, y, random_state<span style="color: #666666">=42</span>)
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> VotingClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
||||
|
||||
log_clf <span style="color: #666666">=</span> LogisticRegression(random_state<span style="color: #666666">=42</span>)
|
||||
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(random_state<span style="color: #666666">=42</span>)
|
||||
svm_clf <span style="color: #666666">=</span> SVC(random_state<span style="color: #666666">=42</span>)
|
||||
|
||||
voting_clf <span style="color: #666666">=</span> VotingClassifier(
|
||||
estimators<span style="color: #666666">=</span>[(<span style="color: #BA2121">'lr'</span>, log_clf), (<span style="color: #BA2121">'rf'</span>, rnd_clf), (<span style="color: #BA2121">'svc'</span>, svm_clf)],
|
||||
voting<span style="color: #666666">=</span><span style="color: #BA2121">'hard'</span>)
|
||||
voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
</pre></div>
|
||||
<p>
|
||||
<b>Bootstrap aggregation</b>, or just <b>bagging</b>, is a
|
||||
general-purpose procedure for reducing the variance of a statistical
|
||||
learning method.
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> clf <span style="color: #AA22FF; font-weight: bold">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(clf<span style="color: #666666">.</span><span style="color: #19177C">__class__</span><span style="color: #666666">.</span><span style="color: #19177C">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>log_clf <span style="color: #666666">=</span> LogisticRegression(random_state<span style="color: #666666">=42</span>)
|
||||
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(random_state<span style="color: #666666">=42</span>)
|
||||
svm_clf <span style="color: #666666">=</span> SVC(probability<span style="color: #666666">=</span><span style="color: #008000">True</span>, random_state<span style="color: #666666">=42</span>)
|
||||
|
||||
voting_clf <span style="color: #666666">=</span> VotingClassifier(
|
||||
estimators<span style="color: #666666">=</span>[(<span style="color: #BA2121">'lr'</span>, log_clf), (<span style="color: #BA2121">'rf'</span>, rnd_clf), (<span style="color: #BA2121">'svc'</span>, svm_clf)],
|
||||
voting<span style="color: #666666">=</span><span style="color: #BA2121">'soft'</span>)
|
||||
voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> clf <span style="color: #AA22FF; font-weight: bold">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(clf<span style="color: #666666">.</span><span style="color: #19177C">__class__</span><span style="color: #666666">.</span><span style="color: #19177C">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -250,6 +218,11 @@ voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
<li><a href="._DecisionTrees-bs031.html">32</a></li>
|
||||
<li><a href="._DecisionTrees-bs032.html">33</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs034.html">35</a></li>
|
||||
<li><a href="._DecisionTrees-bs035.html">36</a></li>
|
||||
<li><a href="._DecisionTrees-bs036.html">37</a></li>
|
||||
<li><a href="._DecisionTrees-bs037.html">38</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs031.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -176,64 +183,32 @@ MathJax.Hub.Config({
|
||||
<a name="part0031"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec30" class="anchor">Bagging examples </h2>
|
||||
<h2 id="___sec30" class="anchor">More bagging </h2>
|
||||
|
||||
<p>
|
||||
Bagging typically results in improved accuracy
|
||||
over prediction using a single tree. Unfortunately, however, it can be
|
||||
difficult to interpret the resulting model. Recall that one of the
|
||||
advantages of decision trees is the attractive and easily interpreted
|
||||
diagram that results.
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> BaggingClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
|
||||
bag_clf <span style="color: #666666">=</span> BaggingClassifier(
|
||||
DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>), n_estimators<span style="color: #666666">=500</span>,
|
||||
max_samples<span style="color: #666666">=100</span>, bootstrap<span style="color: #666666">=</span><span style="color: #008000">True</span>, n_jobs<span style="color: #666666">=-1</span>, random_state<span style="color: #666666">=42</span>)
|
||||
bag_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred <span style="color: #666666">=</span> bag_clf<span style="color: #666666">.</span>predict(X_test)
|
||||
</pre></div>
|
||||
<p>
|
||||
However, when we bag a large number of trees, it is no longer
|
||||
possible to represent the resulting statistical learning procedure
|
||||
using a single tree, and it is no longer clear which variables are
|
||||
most important to the procedure. Thus, bagging improves prediction
|
||||
accuracy at the expense of interpretability. Although the collection
|
||||
of bagged trees is much more difficult to interpret than a single
|
||||
tree, one can obtain an overall summary of the importance of each
|
||||
predictor using the MSE (for bagging regression trees) or the Gini
|
||||
index (for bagging classification trees). In the case of bagging
|
||||
regression trees, we can record the total amount that the MSE is
|
||||
decreased due to splits over a given predictor, averaged over all \( B \) possible
|
||||
trees. A large value indicates an important predictor. Similarly, in
|
||||
the context of bagging classification trees, we can add up the total
|
||||
amount that the Gini index is decreased by splits over a given
|
||||
predictor, averaged over all \( B \) trees.
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
|
||||
<span style="color: #008000; font-weight: bold">print</span>(accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>tree_clf <span style="color: #666666">=</span> DecisionTreeClassifier(random_state<span style="color: #666666">=42</span>)
|
||||
tree_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred_tree <span style="color: #666666">=</span> tree_clf<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(accuracy_score(y_test, y_pred_tree))
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">matplotlib.colors</span> <span style="color: #008000; font-weight: bold">import</span> ListedColormap
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_decision_boundary</span>(clf, X, y, axes<span style="color: #666666">=</span>[<span style="color: #666666">-1.5</span>, <span style="color: #666666">2.5</span>, <span style="color: #666666">-1</span>, <span style="color: #666666">1.5</span>], alpha<span style="color: #666666">=0.5</span>, contour<span style="color: #666666">=</span><span style="color: #008000">True</span>):
|
||||
x1s <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">0</span>], axes[<span style="color: #666666">1</span>], <span style="color: #666666">100</span>)
|
||||
x2s <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(axes[<span style="color: #666666">2</span>], axes[<span style="color: #666666">3</span>], <span style="color: #666666">100</span>)
|
||||
x1, x2 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>meshgrid(x1s, x2s)
|
||||
X_new <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[x1<span style="color: #666666">.</span>ravel(), x2<span style="color: #666666">.</span>ravel()]
|
||||
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_new)<span style="color: #666666">.</span>reshape(x1<span style="color: #666666">.</span>shape)
|
||||
custom_cmap <span style="color: #666666">=</span> ListedColormap([<span style="color: #BA2121">'#fafab0'</span>,<span style="color: #BA2121">'#9898ff'</span>,<span style="color: #BA2121">'#a0faa0'</span>])
|
||||
plt<span style="color: #666666">.</span>contourf(x1, x2, y_pred, alpha<span style="color: #666666">=0.3</span>, cmap<span style="color: #666666">=</span>custom_cmap)
|
||||
<span style="color: #008000; font-weight: bold">if</span> contour:
|
||||
custom_cmap2 <span style="color: #666666">=</span> ListedColormap([<span style="color: #BA2121">'#7d7d58'</span>,<span style="color: #BA2121">'#4c4c7f'</span>,<span style="color: #BA2121">'#507d50'</span>])
|
||||
plt<span style="color: #666666">.</span>contour(x1, x2, y_pred, cmap<span style="color: #666666">=</span>custom_cmap2, alpha<span style="color: #666666">=0.8</span>)
|
||||
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==0</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==0</span>], <span style="color: #BA2121">"yo"</span>, alpha<span style="color: #666666">=</span>alpha)
|
||||
plt<span style="color: #666666">.</span>plot(X[:, <span style="color: #666666">0</span>][y<span style="color: #666666">==1</span>], X[:, <span style="color: #666666">1</span>][y<span style="color: #666666">==1</span>], <span style="color: #BA2121">"bs"</span>, alpha<span style="color: #666666">=</span>alpha)
|
||||
plt<span style="color: #666666">.</span>axis(axes)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r"$x_1$"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r"$x_2$"</span>, fontsize<span style="color: #666666">=18</span>, rotation<span style="color: #666666">=0</span>)
|
||||
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">11</span>,<span style="color: #666666">4</span>))
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">121</span>)
|
||||
plot_decision_boundary(tree_clf, X, y)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"Decision Tree"</span>, fontsize<span style="color: #666666">=14</span>)
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">122</span>)
|
||||
plot_decision_boundary(bag_clf, X, y)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"Decision Trees with Bagging"</span>, fontsize<span style="color: #666666">=14</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -252,6 +227,11 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li class="active"><a href="._DecisionTrees-bs031.html">32</a></li>
|
||||
<li><a href="._DecisionTrees-bs032.html">33</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs034.html">35</a></li>
|
||||
<li><a href="._DecisionTrees-bs035.html">36</a></li>
|
||||
<li><a href="._DecisionTrees-bs036.html">37</a></li>
|
||||
<li><a href="._DecisionTrees-bs037.html">38</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs032.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -67,30 +67,32 @@ Automatically generated HTML file from DocOnce source
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -143,24 +145,29 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">The CART (Classification and Regression Tree) algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Cancer Data again now with Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Then random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Boosting and more</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">Simple Python Code to read in Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs031.html#___sec30" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Simple example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">Bagging Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">A simple scikit-learn example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Please, not the moons again!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Bagging examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Then random forests</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -195,7 +202,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Oct 29, 2019</h4></center> <!-- date -->
|
||||
<center><h4>Oct 31, 2019</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
|
||||
@@ -219,7 +226,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._DecisionTrees-bs008.html">9</a></li>
|
||||
<li><a href="._DecisionTrees-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
||||
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
||||
<li><a href="._DecisionTrees-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -148,7 +148,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p> <br>
|
||||
<center><h4>Oct 29, 2019</h4></center> <!-- date -->
|
||||
<center><h4>Oct 31, 2019</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
|
||||
@@ -627,71 +627,200 @@ $$
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec15">The CART (Classification and Regression Tree) algorithm </h2>
|
||||
<h2 id="___sec15">Visualizing the Tree, Classification </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> confusion_matrix
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> export_graphviz
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">IPython.display</span> <span style="color: #8B008B; font-weight: bold">import</span> Image
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">pydot</span> <span style="color: #8B008B; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
|
||||
|
||||
cancer = load_breast_cancer()
|
||||
X = pd.DataFrame(cancer.data, columns=cancer.feature_names)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(X)
|
||||
y = pd.Categorical.from_codes(cancer.target, cancer.target_names)
|
||||
y = pd.get_dummies(y)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(y)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=<span style="color: #B452CD">1</span>)
|
||||
tree_clf = DecisionTreeClassifier(max_depth=<span style="color: #B452CD">5</span>)
|
||||
tree_clf.fit(X_train, y_train)
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file=<span style="color: #CD5555">"DataFiles/cancer.dot"</span>,
|
||||
feature_names=cancer.feature_names,
|
||||
class_names=cancer.target_names,
|
||||
rounded=<span style="color: #658b00">True</span>,
|
||||
filled=<span style="color: #658b00">True</span>
|
||||
)
|
||||
cmd = <span style="color: #CD5555">'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'</span>
|
||||
os.system(cmd)
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec16">Visualizing the Tree, The Moons </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #228B22"># Common imports</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_moons
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> export_graphviz
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">pydot</span> <span style="color: #8B008B; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span>
|
||||
|
||||
np.random.seed(<span style="color: #B452CD">42</span>)
|
||||
X, y = make_moons(n_samples=<span style="color: #B452CD">100</span>, noise=<span style="color: #B452CD">0.25</span>, random_state=<span style="color: #B452CD">53</span>)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=<span style="color: #B452CD">0</span>)
|
||||
tree_clf = DecisionTreeClassifier(max_depth=<span style="color: #B452CD">5</span>)
|
||||
tree_clf.fit(X_train, y_train)
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file=<span style="color: #CD5555">"DataFiles/moons.dot"</span>,
|
||||
rounded=<span style="color: #658b00">True</span>,
|
||||
filled=<span style="color: #658b00">True</span>
|
||||
)
|
||||
cmd = <span style="color: #CD5555">'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png'</span>
|
||||
os.system(cmd)
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec17">Computing the Gini index </h2>
|
||||
|
||||
<p>
|
||||
The above functions (gini, entropy and misclassification error) are important components of the so-called CART algorithm. We will discuss this algorithm first before we move on to the information gain algorithm ID3.
|
||||
The example we will look at is a classical one in many Machine
|
||||
Learning applications. Based on various meteorological features, we
|
||||
have several so-called attributes which decide whether we at the end
|
||||
will do some outdoor activity like skiing, going for a bike ride etc
|
||||
etc. The table here contains the feautures <b>outlook</b>, <b>temperature</b>,
|
||||
<b>humidity</b> and <b>wind</b>. The target or output is whether we ride
|
||||
(True=1) or whether we do something else that day (False=0). The
|
||||
attributes for each feature are then sunny, overcast and rain for the
|
||||
outlook, hot, cold and mild for temperature, high and normal for
|
||||
humidity and weak and strong for wind.
|
||||
|
||||
<p>
|
||||
The table here summarizes the various attributes and
|
||||
<table border="1">
|
||||
<thead>
|
||||
<tr><th align="center">Day</th> <th align="center">Outlook </th> <th align="center">Temperature</th> <th align="center">Humidity</th> <th align="center"> Wind </th> <th align="center">Ride</th> </tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td align="center"> 1 </td> <td align="center"> Sunny </td> <td align="center"> Hot </td> <td align="center"> High </td> <td align="center"> Weak </td> <td align="center"> 0 </td> </tr>
|
||||
<tr><td align="center"> 2 </td> <td align="center"> Sunny </td> <td align="center"> Hot </td> <td align="center"> High </td> <td align="center"> Strong </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 3 </td> <td align="center"> Overcast </td> <td align="center"> Hot </td> <td align="center"> High </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 4 </td> <td align="center"> Rain </td> <td align="center"> Mild </td> <td align="center"> High </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 5 </td> <td align="center"> Rain </td> <td align="center"> Cool </td> <td align="center"> Normal </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 6 </td> <td align="center"> Rain </td> <td align="center"> Cool </td> <td align="center"> Normal </td> <td align="center"> Strong </td> <td align="center"> 0 </td> </tr>
|
||||
<tr><td align="center"> 7 </td> <td align="center"> Overcast </td> <td align="center"> Cool </td> <td align="center"> Normal </td> <td align="center"> Strong </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 8 </td> <td align="center"> Sunny </td> <td align="center"> Mild </td> <td align="center"> High </td> <td align="center"> Weak </td> <td align="center"> 0 </td> </tr>
|
||||
<tr><td align="center"> 9 </td> <td align="center"> Sunny </td> <td align="center"> Cool </td> <td align="center"> Normal </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 10 </td> <td align="center"> Rain </td> <td align="center"> Mild </td> <td align="center"> Normal </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 11 </td> <td align="center"> Sunny </td> <td align="center"> Mild </td> <td align="center"> Normal </td> <td align="center"> Strong </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 12 </td> <td align="center"> Overcast </td> <td align="center"> Mild </td> <td align="center"> High </td> <td align="center"> Strong </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 13 </td> <td align="center"> Overcast </td> <td align="center"> Hot </td> <td align="center"> Normal </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 14 </td> <td align="center"> Rain </td> <td align="center"> Mild </td> <td align="center"> High </td> <td align="center"> Strong </td> <td align="center"> 0 </td> </tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec18">Simple Python Code to read in Data </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> seed
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> randrange
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">csv</span> <span style="color: #8B008B; font-weight: bold">import</span> reader
|
||||
|
||||
<span style="color: #228B22"># Load a CSV file</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">load_csv</span>(filename):
|
||||
<span style="color: #658b00">file</span> = <span style="color: #658b00">open</span>(filename, <span style="color: #CD5555">"rb"</span>)
|
||||
lines = reader(<span style="color: #658b00">file</span>)
|
||||
dataset = <span style="color: #658b00">list</span>(lines)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> dataset
|
||||
|
||||
<span style="color: #228B22"># Convert string column to float</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">str_column_to_float</span>(dataset, column):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> row <span style="color: #8B008B">in</span> dataset:
|
||||
row[column] = <span style="color: #658b00">float</span>(row[column].strip())
|
||||
|
||||
<span style="color: #228B22"># Split a dataset into k folds</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">cross_validation_split</span>(dataset, n_folds):
|
||||
dataset_split = <span style="color: #658b00">list</span>()
|
||||
dataset_copy = <span style="color: #658b00">list</span>(dataset)
|
||||
fold_size = <span style="color: #658b00">int</span>(<span style="color: #658b00">len</span>(dataset) / n_folds)
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_folds):
|
||||
fold = <span style="color: #658b00">list</span>()
|
||||
<span style="color: #8B008B; font-weight: bold">while</span> <span style="color: #658b00">len</span>(fold) < fold_size:
|
||||
index = randrange(<span style="color: #658b00">len</span>(dataset_copy))
|
||||
fold.append(dataset_copy.pop(index))
|
||||
dataset_split.append(fold)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> dataset_split
|
||||
|
||||
<span style="color: #228B22"># Calculate accuracy percentage</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">accuracy_metric</span>(actual, predicted):
|
||||
correct = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">len</span>(actual)):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> actual[i] == predicted[i]:
|
||||
correct += <span style="color: #B452CD">1</span>
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> correct / <span style="color: #658b00">float</span>(<span style="color: #658b00">len</span>(actual)) * <span style="color: #B452CD">100.0</span>
|
||||
|
||||
<span style="color: #228B22"># Evaluate an algorithm using a cross validation split</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">evaluate_algorithm</span>(dataset, algorithm, n_folds, *args):
|
||||
folds = cross_validation_split(dataset, n_folds)
|
||||
scores = <span style="color: #658b00">list</span>()
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> fold <span style="color: #8B008B">in</span> folds:
|
||||
train_set = <span style="color: #658b00">list</span>(folds)
|
||||
train_set.remove(fold)
|
||||
train_set = <span style="color: #658b00">sum</span>(train_set, [])
|
||||
test_set = <span style="color: #658b00">list</span>()
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> row <span style="color: #8B008B">in</span> fold:
|
||||
row_copy = <span style="color: #658b00">list</span>(row)
|
||||
test_set.append(row_copy)
|
||||
row_copy[-<span style="color: #B452CD">1</span>] = <span style="color: #658b00">None</span>
|
||||
predicted = algorithm(train_set, test_set, *args)
|
||||
actual = [row[-<span style="color: #B452CD">1</span>] <span style="color: #8B008B; font-weight: bold">for</span> row <span style="color: #8B008B">in</span> fold]
|
||||
accuracy = accuracy_metric(actual, predicted)
|
||||
scores.append(accuracy)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> scores
|
||||
|
||||
<span style="color: #228B22"># Split a dataset based on an attribute and an attribute value</span>
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #228B22"># Common imports</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> StandardScaler, OneHotEncoder
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.compose</span> <span style="color: #8B008B; font-weight: bold">import</span> ColumnTransformer
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">IPython.display</span> <span style="color: #8B008B; font-weight: bold">import</span> Image
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">pydot</span> <span style="color: #8B008B; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span>
|
||||
|
||||
<span style="color: #228B22"># Where to save the figures and data files</span>
|
||||
PROJECT_ROOT_DIR = <span style="color: #CD5555">"Results"</span>
|
||||
FIGURE_ID = <span style="color: #CD5555">"Results/FigureFiles"</span>
|
||||
DATA_ID = <span style="color: #CD5555">"DataFiles/"</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #8B008B">not</span> os.path.exists(PROJECT_ROOT_DIR):
|
||||
os.mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #8B008B">not</span> os.path.exists(FIGURE_ID):
|
||||
os.makedirs(FIGURE_ID)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #8B008B">not</span> os.path.exists(DATA_ID):
|
||||
os.makedirs(DATA_ID)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">image_path</span>(fig_id):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> os.path.join(FIGURE_ID, fig_id)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">data_path</span>(dat_id):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> os.path.join(DATA_ID, dat_id)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">save_fig</span>(fig_id):
|
||||
plt.savefig(image_path(fig_id) + <span style="color: #CD5555">".png"</span>, format=<span style="color: #CD5555">'png'</span>)
|
||||
|
||||
infile = <span style="color: #658b00">open</span>(data_path(<span style="color: #CD5555">"ride.csv"</span>),<span style="color: #CD5555">'r'</span>)
|
||||
|
||||
<span style="color: #228B22"># Read the experimental data with Pandas</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">IPython.display</span> <span style="color: #8B008B; font-weight: bold">import</span> display
|
||||
ridedata = pd.read_csv(infile,names = (<span style="color: #CD5555">'Outlook'</span>,<span style="color: #CD5555">'Temperature'</span>,<span style="color: #CD5555">'Humidity'</span>,<span style="color: #CD5555">'Wind'</span>,<span style="color: #CD5555">'Ride'</span>))
|
||||
ridedata = pd.DataFrame(ridedata)
|
||||
display(ridedata)
|
||||
<span style="color: #228B22"># Features and targets</span>
|
||||
X = ridedata.loc[:, ridedata.columns != <span style="color: #CD5555">'Ride'</span>].values
|
||||
display(X)
|
||||
y = ridedata.loc[:, ridedata.columns == <span style="color: #CD5555">'Ride'</span>].values
|
||||
display(y)
|
||||
<span style="color: #228B22"># Categorical variables to one-hot's</span>
|
||||
onehotencoder = OneHotEncoder(categories=<span style="color: #CD5555">"auto"</span>)
|
||||
|
||||
X = ColumnTransformer([(<span style="color: #CD5555">""</span>, onehotencoder)]).fit_transform(X)
|
||||
y.shape
|
||||
|
||||
display(X)
|
||||
display(y)
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec19">Computing the Gini Factor </h2>
|
||||
|
||||
<p>
|
||||
The above functions (gini, entropy and misclassification error) are
|
||||
important components of the so-called CART algorithm. We will discuss
|
||||
this algorithm below after we have discussed the information gain
|
||||
algorithm ID3.
|
||||
|
||||
<p>
|
||||
In the example here we have converted all our attributes into numerical values \( 0,1,2 \) etc.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #228B22"># Split a dataset based on an attribute and an attribute value</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">test_split</span>(index, value, dataset):
|
||||
left, right = <span style="color: #658b00">list</span>(), <span style="color: #658b00">list</span>()
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> row <span style="color: #8B008B">in</span> dataset:
|
||||
@@ -720,7 +849,7 @@ The above functions (gini, entropy and misclassification error) are important co
|
||||
<span style="color: #228B22"># weight the group score by its relative size</span>
|
||||
gini += (<span style="color: #B452CD">1.0</span> - score) * (size / n_instances)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> gini
|
||||
|
||||
|
||||
<span style="color: #228B22"># Select the best split point for a dataset</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">get_split</span>(dataset):
|
||||
class_values = <span style="color: #658b00">list</span>(<span style="color: #658b00">set</span>(row[-<span style="color: #B452CD">1</span>] <span style="color: #8B008B; font-weight: bold">for</span> row <span style="color: #8B008B">in</span> dataset))
|
||||
@@ -729,89 +858,34 @@ The above functions (gini, entropy and misclassification error) are important co
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> row <span style="color: #8B008B">in</span> dataset:
|
||||
groups = test_split(index, row[index], dataset)
|
||||
gini = gini_index(groups, class_values)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">'X%d < %.3f Gini=%.3f'</span> % ((index+<span style="color: #B452CD">1</span>), row[index], gini))
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> gini < b_score:
|
||||
b_index, b_value, b_score, b_groups = index, row[index], gini, groups
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> {<span style="color: #CD5555">'index'</span>:b_index, <span style="color: #CD5555">'value'</span>:b_value, <span style="color: #CD5555">'groups'</span>:b_groups}
|
||||
|
||||
<span style="color: #228B22"># Create a terminal node value</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">to_terminal</span>(group):
|
||||
outcomes = [row[-<span style="color: #B452CD">1</span>] <span style="color: #8B008B; font-weight: bold">for</span> row <span style="color: #8B008B">in</span> group]
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #658b00">max</span>(<span style="color: #658b00">set</span>(outcomes), key=outcomes.count)
|
||||
|
||||
<span style="color: #228B22"># Create child splits for a node or make terminal</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">split</span>(node, max_depth, min_size, depth):
|
||||
left, right = node[<span style="color: #CD5555">'groups'</span>]
|
||||
<span style="color: #8B008B; font-weight: bold">del</span>(node[<span style="color: #CD5555">'groups'</span>])
|
||||
<span style="color: #228B22"># check for a no split</span>
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #8B008B">not</span> left <span style="color: #8B008B">or</span> <span style="color: #8B008B">not</span> right:
|
||||
node[<span style="color: #CD5555">'left'</span>] = node[<span style="color: #CD5555">'right'</span>] = to_terminal(left + right)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span>
|
||||
<span style="color: #228B22"># check for max depth</span>
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> depth >= max_depth:
|
||||
node[<span style="color: #CD5555">'left'</span>], node[<span style="color: #CD5555">'right'</span>] = to_terminal(left), to_terminal(right)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span>
|
||||
<span style="color: #228B22"># process left child</span>
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #658b00">len</span>(left) <= min_size:
|
||||
node[<span style="color: #CD5555">'left'</span>] = to_terminal(left)
|
||||
<span style="color: #8B008B; font-weight: bold">else</span>:
|
||||
node[<span style="color: #CD5555">'left'</span>] = get_split(left)
|
||||
split(node[<span style="color: #CD5555">'left'</span>], max_depth, min_size, depth+<span style="color: #B452CD">1</span>)
|
||||
<span style="color: #228B22"># process right child</span>
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #658b00">len</span>(right) <= min_size:
|
||||
node[<span style="color: #CD5555">'right'</span>] = to_terminal(right)
|
||||
<span style="color: #8B008B; font-weight: bold">else</span>:
|
||||
node[<span style="color: #CD5555">'right'</span>] = get_split(right)
|
||||
split(node[<span style="color: #CD5555">'right'</span>], max_depth, min_size, depth+<span style="color: #B452CD">1</span>)
|
||||
|
||||
<span style="color: #228B22"># Build a decision tree</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">build_tree</span>(train, max_depth, min_size):
|
||||
root = get_split(train)
|
||||
split(root, max_depth, min_size, <span style="color: #B452CD">1</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> root
|
||||
|
||||
<span style="color: #228B22"># Make a prediction with a decision tree</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">predict</span>(node, row):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> row[node[<span style="color: #CD5555">'index'</span>]] < node[<span style="color: #CD5555">'value'</span>]:
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #658b00">isinstance</span>(node[<span style="color: #CD5555">'left'</span>], <span style="color: #658b00">dict</span>):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> predict(node[<span style="color: #CD5555">'left'</span>], row)
|
||||
<span style="color: #8B008B; font-weight: bold">else</span>:
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> node[<span style="color: #CD5555">'left'</span>]
|
||||
<span style="color: #8B008B; font-weight: bold">else</span>:
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #658b00">isinstance</span>(node[<span style="color: #CD5555">'right'</span>], <span style="color: #658b00">dict</span>):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> predict(node[<span style="color: #CD5555">'right'</span>], row)
|
||||
<span style="color: #8B008B; font-weight: bold">else</span>:
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> node[<span style="color: #CD5555">'right'</span>]
|
||||
|
||||
<span style="color: #228B22"># Classification and Regression Tree Algorithm</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">decision_tree</span>(train, test, max_depth, min_size):
|
||||
tree = build_tree(train, max_depth, min_size)
|
||||
predictions = <span style="color: #658b00">list</span>()
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> row <span style="color: #8B008B">in</span> test:
|
||||
prediction = predict(tree, row)
|
||||
predictions.append(prediction)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span>(predictions)
|
||||
|
||||
<span style="color: #228B22"># Test CART </span>
|
||||
seed(<span style="color: #B452CD">1</span>)
|
||||
<span style="color: #228B22"># load and prepare data</span>
|
||||
filename = <span style="color: #CD5555">'DataFiles/rideclass.csv'</span>
|
||||
dataset = load_csv(filename)
|
||||
<span style="color: #228B22"># convert string attributes to integers</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">len</span>(dataset[<span style="color: #B452CD">0</span>])):
|
||||
str_column_to_float(dataset, i)
|
||||
<span style="color: #228B22"># evaluate algorithm</span>
|
||||
n_folds = <span style="color: #B452CD">5</span>
|
||||
max_depth = <span style="color: #B452CD">5</span>
|
||||
min_size = <span style="color: #B452CD">10</span>
|
||||
scores = evaluate_algorithm(dataset, decision_tree, n_folds, max_depth, min_size)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">'Scores: %s'</span> % scores)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">'Mean Accuracy: %.3f%%'</span> % (<span style="color: #658b00">sum</span>(scores)/<span style="color: #658b00">float</span>(<span style="color: #658b00">len</span>(scores))))
|
||||
dataset = [[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>],
|
||||
[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">2</span>,<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">2</span>,<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>],
|
||||
[<span style="color: #B452CD">1</span>,<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>],
|
||||
[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>]]
|
||||
|
||||
split = get_split(dataset)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">'Split: [X%d < %.3f]'</span> % ((split[<span style="color: #CD5555">'index'</span>]+<span style="color: #B452CD">1</span>), split[<span style="color: #CD5555">'value'</span>]))
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec16">Entropy and the ID3 algorithm </h2>
|
||||
<h2 id="___sec20">Entropy and the ID3 algorithm </h2>
|
||||
|
||||
<p>
|
||||
ID3, learns decision trees by constructing
|
||||
@@ -848,15 +922,216 @@ attributes at each step while growing the tree.
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec17">Implementing the ID3 Algorithm </h2>
|
||||
<h2 id="___sec21">Implementing the ID3 Algorithm </h2>
|
||||
|
||||
<p>
|
||||
<b>more text to come here</b>, material presented during lecture Friday Oct 25.
|
||||
import re
|
||||
import math
|
||||
from collections import deque
|
||||
|
||||
<p>
|
||||
<!-- x is examples in training set -->
|
||||
<!-- y is set of attributes -->
|
||||
<!-- label is target attributes -->
|
||||
<!-- Node is a class which has properties values, childs, and next -->
|
||||
<!-- root is top node in the decision tree -->
|
||||
|
||||
<p>
|
||||
class Node(object):
|
||||
def __init__(self):
|
||||
self.value = None
|
||||
self.next = None
|
||||
self.childs = None
|
||||
|
||||
<p>
|
||||
<!-- Simple class of Decision Tree -->
|
||||
<!-- Aimed for who want to learn Decision Tree, so it is not optimized -->
|
||||
class DecisionTree(object):
|
||||
def __init__(self, sample, attributes, labels):
|
||||
self.sample = sample
|
||||
self.attributes = attributes
|
||||
self.labels = labels
|
||||
self.labelCodes = None
|
||||
self.labelCodesCount = None
|
||||
self.initLabelCodes()
|
||||
# print(self.labelCodes)
|
||||
self.root = None
|
||||
self.entropy = self.getEntropy([x for x in range(len(self.labels))])
|
||||
|
||||
<p>
|
||||
def initLabelCodes(self):
|
||||
self.labelCodes = []
|
||||
self.labelCodesCount = []
|
||||
for l in self.labels:
|
||||
if l not in self.labelCodes:
|
||||
self.labelCodes.append(l)
|
||||
self.labelCodesCount.append(0)
|
||||
self.labelCodesCount[self.labelCodes.index(l)] += 1
|
||||
|
||||
<p>
|
||||
def getLabelCodeId(self, sampleId):
|
||||
return self.labelCodes.index(self.labels[sampleId])
|
||||
|
||||
<p>
|
||||
def getAttributeValues(self, sampleIds, attributeId):
|
||||
vals = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in vals:
|
||||
vals.append(val)
|
||||
# print(vals)
|
||||
return vals
|
||||
|
||||
<p>
|
||||
def getEntropy(self, sampleIds):
|
||||
entropy = 0
|
||||
labelCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCount[self.getLabelCodeId(sid)] += 1
|
||||
# print("-ge", labelCount)
|
||||
for lv in labelCount:
|
||||
# print(lv)
|
||||
if lv != 0:
|
||||
entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
|
||||
else:
|
||||
entropy += 0
|
||||
return entropy
|
||||
|
||||
<p>
|
||||
def getDominantLabel(self, sampleIds):
|
||||
labelCodesCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
|
||||
return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
|
||||
|
||||
<p>
|
||||
def getInformationGain(self, sampleIds, attributeId):
|
||||
gain = self.getEntropy(sampleIds)
|
||||
attributeVals = []
|
||||
attributeValsCount = []
|
||||
attributeValsIds = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in attributeVals:
|
||||
attributeVals.append(val)
|
||||
attributeValsCount.append(0)
|
||||
attributeValsIds.append([])
|
||||
vid = attributeVals.index(val)
|
||||
attributeValsCount[vid] += 1
|
||||
attributeValsIds[vid].append(sid)
|
||||
# print("-gig", self.attributes[attributeId])
|
||||
for vc, vids in zip(attributeValsCount, attributeValsIds):
|
||||
# print("-gig", vids)
|
||||
gain -= vc/len(sampleIds) * self.getEntropy(vids)
|
||||
return gain
|
||||
|
||||
<p>
|
||||
def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
|
||||
attributesEntropy = [0] * len(attributeIds)
|
||||
for i, attId in zip(range(len(attributeIds)), attributeIds):
|
||||
attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
|
||||
maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
|
||||
return self.attributes[maxId], maxId
|
||||
|
||||
<p>
|
||||
def isSingleLabeled(self, sampleIds):
|
||||
label = self.labels[sampleIds[0]]
|
||||
for sid in sampleIds:
|
||||
if self.labels[sid] != label:
|
||||
return False
|
||||
return True
|
||||
|
||||
<p>
|
||||
def getLabel(self, sampleId):
|
||||
return self.labels[sampleId]
|
||||
|
||||
<p>
|
||||
def id3(self):
|
||||
sampleIds = [x for x in range(len(self.sample))]
|
||||
attributeIds = [x for x in range(len(self.attributes))]
|
||||
self.root = self.id3Recv(sampleIds, attributeIds, self.root)
|
||||
|
||||
<p>
|
||||
def id3Recv(self, sampleIds, attributeIds, root):
|
||||
root = Node() # Initialize current root
|
||||
if self.isSingleLabeled(sampleIds):
|
||||
root.value = self.labels[sampleIds[0]]
|
||||
return root
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) == 0:
|
||||
root.value = self.getDominantLabel(sampleIds)
|
||||
return root
|
||||
bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
|
||||
sampleIds, attributeIds)
|
||||
# print(bestAttrName)
|
||||
root.value = bestAttrName
|
||||
root.childs = [] # Create list of children
|
||||
for value in self.getAttributeValues(sampleIds, bestAttrId):
|
||||
# print(value)
|
||||
child = Node()
|
||||
child.value = value
|
||||
root.childs.append(child) # Append new child node to current
|
||||
# root
|
||||
childSampleIds = []
|
||||
for sid in sampleIds:
|
||||
if self.sample[sid][bestAttrId] == value:
|
||||
childSampleIds.append(sid)
|
||||
if len(childSampleIds) == 0:
|
||||
child.next = self.getDominantLabel(sampleIds)
|
||||
else:
|
||||
# print(bestAttrName, bestAttrId)
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) > 0 and bestAttrId in attributeIds:
|
||||
toRemove = attributeIds.index(bestAttrId)
|
||||
attributeIds.pop(toRemove)
|
||||
child.next = self.id3Recv(
|
||||
childSampleIds, attributeIds, child.next)
|
||||
return root
|
||||
|
||||
<p>
|
||||
def printTree(self):
|
||||
if self.root:
|
||||
roots = deque()
|
||||
roots.append(self.root)
|
||||
while len(roots) > 0:
|
||||
root = roots.popleft()
|
||||
print(root.value)
|
||||
if root.childs:
|
||||
for child in root.childs:
|
||||
print('({})'.format(child.value))
|
||||
roots.append(child.next)
|
||||
elif root.next:
|
||||
print(root.next)
|
||||
|
||||
<p>
|
||||
def test():
|
||||
f = open('DataFiles/rideclass.csv')
|
||||
attributes = f.readline().split(',')
|
||||
attributes = attributes[1:len(attributes)-1]
|
||||
print(attributes)
|
||||
sample = f.readlines()
|
||||
f.close()
|
||||
for i in range(len(sample)):
|
||||
sample[i] = re.sub('\d+,', '', sample[i])
|
||||
sample[i] = sample[i].strip().split(',')
|
||||
labels = []
|
||||
for s in sample:
|
||||
labels.append(s.pop())
|
||||
# print(sample)
|
||||
# print(labels)
|
||||
decisionTree = DecisionTree(sample, attributes, labels)
|
||||
print("System entropy {}".format(decisionTree.entropy))
|
||||
decisionTree.id3()
|
||||
decisionTree.printTree()
|
||||
|
||||
<p>
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec18">Cancer Data again now with Decision Trees </h2>
|
||||
<h2 id="___sec22">Cancer Data again now with Decision Trees and other Methods </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -906,7 +1181,7 @@ deep_tree_clf.fit(X_train_scaled, y_train)
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec19">Another example, the moons again </h2>
|
||||
<h2 id="___sec23">Another example, the moons again </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -979,7 +1254,7 @@ plt.show()
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec20">Playing around with regions </h2>
|
||||
<h2 id="___sec24">Playing around with regions </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -987,7 +1262,7 @@ plt.show()
|
||||
Xs = np.random.rand(<span style="color: #B452CD">100</span>, <span style="color: #B452CD">2</span>) - <span style="color: #B452CD">0.5</span>
|
||||
ys = (Xs[:, <span style="color: #B452CD">0</span>] > <span style="color: #B452CD">0</span>).astype(np.float32) * <span style="color: #B452CD">2</span>
|
||||
|
||||
angle = np.pi / <span style="color: #B452CD">4</span>
|
||||
angle = np.pi/<span style="color: #B452CD">4</span>
|
||||
rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
|
||||
Xsr = Xs.dot(rotation_matrix)
|
||||
|
||||
@@ -1008,7 +1283,7 @@ plt.show()
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec21">Regression trees </h2>
|
||||
<h2 id="___sec25">Regression trees </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -1031,7 +1306,7 @@ tree_reg.fit(X, y)
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec22">Final regressor code </h2>
|
||||
<h2 id="___sec26">Final regressor code </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -1110,7 +1385,7 @@ plt.show()
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec23">Pros and cons of trees, pros </h2>
|
||||
<h2 id="___sec27">Pros and cons of trees, pros </h2>
|
||||
|
||||
<ul>
|
||||
<p><li> White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines)</li>
|
||||
@@ -1125,7 +1400,7 @@ plt.show()
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec24">Disadvantages </h2>
|
||||
<h2 id="___sec28">Disadvantages </h2>
|
||||
|
||||
<ul>
|
||||
<p><li> Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches</li>
|
||||
@@ -1143,7 +1418,7 @@ However, by aggregating many decision trees, using methods like bagging, random
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec25">Bagging </h2>
|
||||
<h2 id="___sec29">Bagging </h2>
|
||||
|
||||
<p>
|
||||
The <b>plain</b> decision trees suffer from high
|
||||
@@ -1158,6 +1433,11 @@ of \( n \) to \( p \) is moderately large.
|
||||
<b>Bootstrap aggregation</b>, or just <b>bagging</b>, is a
|
||||
general-purpose procedure for reducing the variance of a statistical
|
||||
learning method.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec30">More bagging </h2>
|
||||
|
||||
<p>
|
||||
Bagging typically results in improved accuracy
|
||||
@@ -1186,7 +1466,7 @@ predictor, averaged over all \( B \) trees.
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec26">Simple example, head or tail </h2>
|
||||
<h2 id="___sec31">Simple example, head or tail </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -1207,7 +1487,59 @@ plt.show()
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec27">Random forests </h2>
|
||||
<h2 id="___sec32">Bagging Example </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_moons
|
||||
|
||||
X, y = make_moons(n_samples=<span style="color: #B452CD">500</span>, noise=<span style="color: #B452CD">0.30</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> VotingClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||||
|
||||
log_clf = LogisticRegression(solver=<span style="color: #CD5555">"liblinear"</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf = RandomForestClassifier(n_estimators=<span style="color: #B452CD">10</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
svm_clf = SVC(gamma=<span style="color: #CD5555">"auto"</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[(<span style="color: #CD5555">'lr'</span>, log_clf), (<span style="color: #CD5555">'rf'</span>, rnd_clf), (<span style="color: #CD5555">'svc'</span>, svm_clf)],
|
||||
voting=<span style="color: #CD5555">'hard'</span>)
|
||||
|
||||
voting_clf.fit(X_train, y_train)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> clf <span style="color: #8B008B">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(clf.<span style="color: #00688B">__class__</span>.<span style="color: #00688B">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
|
||||
log_clf = LogisticRegression(solver=<span style="color: #CD5555">"liblinear"</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf = RandomForestClassifier(n_estimators=<span style="color: #B452CD">10</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
svm_clf = SVC(gamma=<span style="color: #CD5555">"auto"</span>, probability=<span style="color: #658b00">True</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[(<span style="color: #CD5555">'lr'</span>, log_clf), (<span style="color: #CD5555">'rf'</span>, rnd_clf), (<span style="color: #CD5555">'svc'</span>, svm_clf)],
|
||||
voting=<span style="color: #CD5555">'soft'</span>)
|
||||
voting_clf.fit(X_train, y_train)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> clf <span style="color: #8B008B">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(clf.<span style="color: #00688B">__class__</span>.<span style="color: #00688B">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec33">Random forests </h2>
|
||||
|
||||
<p>
|
||||
Random forests provide an improvement over bagged trees by way of a
|
||||
@@ -1253,7 +1585,7 @@ this setting.
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec28">A simple scikit-learn example </h2>
|
||||
<h2 id="___sec34">A simple scikit-learn example </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -1272,7 +1604,7 @@ accuracy = cross_validate(Random_Forest_model,X,Y,cv=<span style="color: #B452CD
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec29">Please, not the moons again! </h2>
|
||||
<h2 id="___sec35">Please, not the moons again! </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -1331,7 +1663,7 @@ voting_clf.fit(X_train, y_train)
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec30">Bagging examples </h2>
|
||||
<h2 id="___sec36">Bagging examples </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -1393,7 +1725,7 @@ plt.show()
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec31">Then random forests </h2>
|
||||
<h2 id="___sec37">Then random forests </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -1415,12 +1747,6 @@ np.sum(y_pred == y_pred_rf) / <span style="color: #658b00">len</span>(y_pred)
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec32">Boosting and more </h2>
|
||||
More material to come here.
|
||||
</section>
|
||||
|
||||
|
||||
|
||||
</div> <!-- class="slides" -->
|
||||
</div> <!-- class="reveal" -->
|
||||
|
||||
@@ -87,30 +87,32 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -152,7 +154,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Oct 29, 2019</h4></center> <!-- date -->
|
||||
<center><h4>Oct 31, 2019</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
@@ -606,71 +608,196 @@ $$
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec15">The CART (Classification and Regression Tree) algorithm </h2>
|
||||
<h2 id="___sec15">Visualizing the Tree, Classification </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> confusion_matrix
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> export_graphviz
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">IPython.display</span> <span style="color: #8B008B; font-weight: bold">import</span> Image
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">pydot</span> <span style="color: #8B008B; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
|
||||
|
||||
cancer = load_breast_cancer()
|
||||
X = pd.DataFrame(cancer.data, columns=cancer.feature_names)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(X)
|
||||
y = pd.Categorical.from_codes(cancer.target, cancer.target_names)
|
||||
y = pd.get_dummies(y)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(y)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=<span style="color: #B452CD">1</span>)
|
||||
tree_clf = DecisionTreeClassifier(max_depth=<span style="color: #B452CD">5</span>)
|
||||
tree_clf.fit(X_train, y_train)
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file=<span style="color: #CD5555">"DataFiles/cancer.dot"</span>,
|
||||
feature_names=cancer.feature_names,
|
||||
class_names=cancer.target_names,
|
||||
rounded=<span style="color: #658b00">True</span>,
|
||||
filled=<span style="color: #658b00">True</span>
|
||||
)
|
||||
cmd = <span style="color: #CD5555">'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'</span>
|
||||
os.system(cmd)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec16">Visualizing the Tree, The Moons </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #228B22"># Common imports</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_moons
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> export_graphviz
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">pydot</span> <span style="color: #8B008B; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span>
|
||||
|
||||
np.random.seed(<span style="color: #B452CD">42</span>)
|
||||
X, y = make_moons(n_samples=<span style="color: #B452CD">100</span>, noise=<span style="color: #B452CD">0.25</span>, random_state=<span style="color: #B452CD">53</span>)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=<span style="color: #B452CD">0</span>)
|
||||
tree_clf = DecisionTreeClassifier(max_depth=<span style="color: #B452CD">5</span>)
|
||||
tree_clf.fit(X_train, y_train)
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file=<span style="color: #CD5555">"DataFiles/moons.dot"</span>,
|
||||
rounded=<span style="color: #658b00">True</span>,
|
||||
filled=<span style="color: #658b00">True</span>
|
||||
)
|
||||
cmd = <span style="color: #CD5555">'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png'</span>
|
||||
os.system(cmd)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec17">Computing the Gini index </h2>
|
||||
|
||||
<p>
|
||||
The above functions (gini, entropy and misclassification error) are important components of the so-called CART algorithm. We will discuss this algorithm first before we move on to the information gain algorithm ID3.
|
||||
The example we will look at is a classical one in many Machine
|
||||
Learning applications. Based on various meteorological features, we
|
||||
have several so-called attributes which decide whether we at the end
|
||||
will do some outdoor activity like skiing, going for a bike ride etc
|
||||
etc. The table here contains the feautures <b>outlook</b>, <b>temperature</b>,
|
||||
<b>humidity</b> and <b>wind</b>. The target or output is whether we ride
|
||||
(True=1) or whether we do something else that day (False=0). The
|
||||
attributes for each feature are then sunny, overcast and rain for the
|
||||
outlook, hot, cold and mild for temperature, high and normal for
|
||||
humidity and weak and strong for wind.
|
||||
|
||||
<p>
|
||||
The table here summarizes the various attributes and
|
||||
<table border="1">
|
||||
<thead>
|
||||
<tr><th align="center">Day</th> <th align="center">Outlook </th> <th align="center">Temperature</th> <th align="center">Humidity</th> <th align="center"> Wind </th> <th align="center">Ride</th> </tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td align="center"> 1 </td> <td align="center"> Sunny </td> <td align="center"> Hot </td> <td align="center"> High </td> <td align="center"> Weak </td> <td align="center"> 0 </td> </tr>
|
||||
<tr><td align="center"> 2 </td> <td align="center"> Sunny </td> <td align="center"> Hot </td> <td align="center"> High </td> <td align="center"> Strong </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 3 </td> <td align="center"> Overcast </td> <td align="center"> Hot </td> <td align="center"> High </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 4 </td> <td align="center"> Rain </td> <td align="center"> Mild </td> <td align="center"> High </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 5 </td> <td align="center"> Rain </td> <td align="center"> Cool </td> <td align="center"> Normal </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 6 </td> <td align="center"> Rain </td> <td align="center"> Cool </td> <td align="center"> Normal </td> <td align="center"> Strong </td> <td align="center"> 0 </td> </tr>
|
||||
<tr><td align="center"> 7 </td> <td align="center"> Overcast </td> <td align="center"> Cool </td> <td align="center"> Normal </td> <td align="center"> Strong </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 8 </td> <td align="center"> Sunny </td> <td align="center"> Mild </td> <td align="center"> High </td> <td align="center"> Weak </td> <td align="center"> 0 </td> </tr>
|
||||
<tr><td align="center"> 9 </td> <td align="center"> Sunny </td> <td align="center"> Cool </td> <td align="center"> Normal </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 10 </td> <td align="center"> Rain </td> <td align="center"> Mild </td> <td align="center"> Normal </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 11 </td> <td align="center"> Sunny </td> <td align="center"> Mild </td> <td align="center"> Normal </td> <td align="center"> Strong </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 12 </td> <td align="center"> Overcast </td> <td align="center"> Mild </td> <td align="center"> High </td> <td align="center"> Strong </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 13 </td> <td align="center"> Overcast </td> <td align="center"> Hot </td> <td align="center"> Normal </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 14 </td> <td align="center"> Rain </td> <td align="center"> Mild </td> <td align="center"> High </td> <td align="center"> Strong </td> <td align="center"> 0 </td> </tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec18">Simple Python Code to read in Data </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> seed
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> randrange
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">csv</span> <span style="color: #8B008B; font-weight: bold">import</span> reader
|
||||
|
||||
<span style="color: #228B22"># Load a CSV file</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">load_csv</span>(filename):
|
||||
<span style="color: #658b00">file</span> = <span style="color: #658b00">open</span>(filename, <span style="color: #CD5555">"rb"</span>)
|
||||
lines = reader(<span style="color: #658b00">file</span>)
|
||||
dataset = <span style="color: #658b00">list</span>(lines)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> dataset
|
||||
|
||||
<span style="color: #228B22"># Convert string column to float</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">str_column_to_float</span>(dataset, column):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> row <span style="color: #8B008B">in</span> dataset:
|
||||
row[column] = <span style="color: #658b00">float</span>(row[column].strip())
|
||||
|
||||
<span style="color: #228B22"># Split a dataset into k folds</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">cross_validation_split</span>(dataset, n_folds):
|
||||
dataset_split = <span style="color: #658b00">list</span>()
|
||||
dataset_copy = <span style="color: #658b00">list</span>(dataset)
|
||||
fold_size = <span style="color: #658b00">int</span>(<span style="color: #658b00">len</span>(dataset) / n_folds)
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_folds):
|
||||
fold = <span style="color: #658b00">list</span>()
|
||||
<span style="color: #8B008B; font-weight: bold">while</span> <span style="color: #658b00">len</span>(fold) < fold_size:
|
||||
index = randrange(<span style="color: #658b00">len</span>(dataset_copy))
|
||||
fold.append(dataset_copy.pop(index))
|
||||
dataset_split.append(fold)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> dataset_split
|
||||
|
||||
<span style="color: #228B22"># Calculate accuracy percentage</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">accuracy_metric</span>(actual, predicted):
|
||||
correct = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">len</span>(actual)):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> actual[i] == predicted[i]:
|
||||
correct += <span style="color: #B452CD">1</span>
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> correct / <span style="color: #658b00">float</span>(<span style="color: #658b00">len</span>(actual)) * <span style="color: #B452CD">100.0</span>
|
||||
|
||||
<span style="color: #228B22"># Evaluate an algorithm using a cross validation split</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">evaluate_algorithm</span>(dataset, algorithm, n_folds, *args):
|
||||
folds = cross_validation_split(dataset, n_folds)
|
||||
scores = <span style="color: #658b00">list</span>()
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> fold <span style="color: #8B008B">in</span> folds:
|
||||
train_set = <span style="color: #658b00">list</span>(folds)
|
||||
train_set.remove(fold)
|
||||
train_set = <span style="color: #658b00">sum</span>(train_set, [])
|
||||
test_set = <span style="color: #658b00">list</span>()
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> row <span style="color: #8B008B">in</span> fold:
|
||||
row_copy = <span style="color: #658b00">list</span>(row)
|
||||
test_set.append(row_copy)
|
||||
row_copy[-<span style="color: #B452CD">1</span>] = <span style="color: #658b00">None</span>
|
||||
predicted = algorithm(train_set, test_set, *args)
|
||||
actual = [row[-<span style="color: #B452CD">1</span>] <span style="color: #8B008B; font-weight: bold">for</span> row <span style="color: #8B008B">in</span> fold]
|
||||
accuracy = accuracy_metric(actual, predicted)
|
||||
scores.append(accuracy)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> scores
|
||||
|
||||
<span style="color: #228B22"># Split a dataset based on an attribute and an attribute value</span>
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #228B22"># Common imports</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> StandardScaler, OneHotEncoder
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.compose</span> <span style="color: #8B008B; font-weight: bold">import</span> ColumnTransformer
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">IPython.display</span> <span style="color: #8B008B; font-weight: bold">import</span> Image
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">pydot</span> <span style="color: #8B008B; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span>
|
||||
|
||||
<span style="color: #228B22"># Where to save the figures and data files</span>
|
||||
PROJECT_ROOT_DIR = <span style="color: #CD5555">"Results"</span>
|
||||
FIGURE_ID = <span style="color: #CD5555">"Results/FigureFiles"</span>
|
||||
DATA_ID = <span style="color: #CD5555">"DataFiles/"</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #8B008B">not</span> os.path.exists(PROJECT_ROOT_DIR):
|
||||
os.mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #8B008B">not</span> os.path.exists(FIGURE_ID):
|
||||
os.makedirs(FIGURE_ID)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #8B008B">not</span> os.path.exists(DATA_ID):
|
||||
os.makedirs(DATA_ID)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">image_path</span>(fig_id):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> os.path.join(FIGURE_ID, fig_id)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">data_path</span>(dat_id):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> os.path.join(DATA_ID, dat_id)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">save_fig</span>(fig_id):
|
||||
plt.savefig(image_path(fig_id) + <span style="color: #CD5555">".png"</span>, format=<span style="color: #CD5555">'png'</span>)
|
||||
|
||||
infile = <span style="color: #658b00">open</span>(data_path(<span style="color: #CD5555">"ride.csv"</span>),<span style="color: #CD5555">'r'</span>)
|
||||
|
||||
<span style="color: #228B22"># Read the experimental data with Pandas</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">IPython.display</span> <span style="color: #8B008B; font-weight: bold">import</span> display
|
||||
ridedata = pd.read_csv(infile,names = (<span style="color: #CD5555">'Outlook'</span>,<span style="color: #CD5555">'Temperature'</span>,<span style="color: #CD5555">'Humidity'</span>,<span style="color: #CD5555">'Wind'</span>,<span style="color: #CD5555">'Ride'</span>))
|
||||
ridedata = pd.DataFrame(ridedata)
|
||||
display(ridedata)
|
||||
<span style="color: #228B22"># Features and targets</span>
|
||||
X = ridedata.loc[:, ridedata.columns != <span style="color: #CD5555">'Ride'</span>].values
|
||||
display(X)
|
||||
y = ridedata.loc[:, ridedata.columns == <span style="color: #CD5555">'Ride'</span>].values
|
||||
display(y)
|
||||
<span style="color: #228B22"># Categorical variables to one-hot's</span>
|
||||
onehotencoder = OneHotEncoder(categories=<span style="color: #CD5555">"auto"</span>)
|
||||
|
||||
X = ColumnTransformer([(<span style="color: #CD5555">""</span>, onehotencoder)]).fit_transform(X)
|
||||
y.shape
|
||||
|
||||
display(X)
|
||||
display(y)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec19">Computing the Gini Factor </h2>
|
||||
|
||||
<p>
|
||||
The above functions (gini, entropy and misclassification error) are
|
||||
important components of the so-called CART algorithm. We will discuss
|
||||
this algorithm below after we have discussed the information gain
|
||||
algorithm ID3.
|
||||
|
||||
<p>
|
||||
In the example here we have converted all our attributes into numerical values \( 0,1,2 \) etc.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #228B22"># Split a dataset based on an attribute and an attribute value</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">test_split</span>(index, value, dataset):
|
||||
left, right = <span style="color: #658b00">list</span>(), <span style="color: #658b00">list</span>()
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> row <span style="color: #8B008B">in</span> dataset:
|
||||
@@ -699,7 +826,7 @@ The above functions (gini, entropy and misclassification error) are important co
|
||||
<span style="color: #228B22"># weight the group score by its relative size</span>
|
||||
gini += (<span style="color: #B452CD">1.0</span> - score) * (size / n_instances)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> gini
|
||||
|
||||
|
||||
<span style="color: #228B22"># Select the best split point for a dataset</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">get_split</span>(dataset):
|
||||
class_values = <span style="color: #658b00">list</span>(<span style="color: #658b00">set</span>(row[-<span style="color: #B452CD">1</span>] <span style="color: #8B008B; font-weight: bold">for</span> row <span style="color: #8B008B">in</span> dataset))
|
||||
@@ -708,88 +835,33 @@ The above functions (gini, entropy and misclassification error) are important co
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> row <span style="color: #8B008B">in</span> dataset:
|
||||
groups = test_split(index, row[index], dataset)
|
||||
gini = gini_index(groups, class_values)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">'X%d < %.3f Gini=%.3f'</span> % ((index+<span style="color: #B452CD">1</span>), row[index], gini))
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> gini < b_score:
|
||||
b_index, b_value, b_score, b_groups = index, row[index], gini, groups
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> {<span style="color: #CD5555">'index'</span>:b_index, <span style="color: #CD5555">'value'</span>:b_value, <span style="color: #CD5555">'groups'</span>:b_groups}
|
||||
|
||||
<span style="color: #228B22"># Create a terminal node value</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">to_terminal</span>(group):
|
||||
outcomes = [row[-<span style="color: #B452CD">1</span>] <span style="color: #8B008B; font-weight: bold">for</span> row <span style="color: #8B008B">in</span> group]
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #658b00">max</span>(<span style="color: #658b00">set</span>(outcomes), key=outcomes.count)
|
||||
|
||||
<span style="color: #228B22"># Create child splits for a node or make terminal</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">split</span>(node, max_depth, min_size, depth):
|
||||
left, right = node[<span style="color: #CD5555">'groups'</span>]
|
||||
<span style="color: #8B008B; font-weight: bold">del</span>(node[<span style="color: #CD5555">'groups'</span>])
|
||||
<span style="color: #228B22"># check for a no split</span>
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #8B008B">not</span> left <span style="color: #8B008B">or</span> <span style="color: #8B008B">not</span> right:
|
||||
node[<span style="color: #CD5555">'left'</span>] = node[<span style="color: #CD5555">'right'</span>] = to_terminal(left + right)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span>
|
||||
<span style="color: #228B22"># check for max depth</span>
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> depth >= max_depth:
|
||||
node[<span style="color: #CD5555">'left'</span>], node[<span style="color: #CD5555">'right'</span>] = to_terminal(left), to_terminal(right)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span>
|
||||
<span style="color: #228B22"># process left child</span>
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #658b00">len</span>(left) <= min_size:
|
||||
node[<span style="color: #CD5555">'left'</span>] = to_terminal(left)
|
||||
<span style="color: #8B008B; font-weight: bold">else</span>:
|
||||
node[<span style="color: #CD5555">'left'</span>] = get_split(left)
|
||||
split(node[<span style="color: #CD5555">'left'</span>], max_depth, min_size, depth+<span style="color: #B452CD">1</span>)
|
||||
<span style="color: #228B22"># process right child</span>
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #658b00">len</span>(right) <= min_size:
|
||||
node[<span style="color: #CD5555">'right'</span>] = to_terminal(right)
|
||||
<span style="color: #8B008B; font-weight: bold">else</span>:
|
||||
node[<span style="color: #CD5555">'right'</span>] = get_split(right)
|
||||
split(node[<span style="color: #CD5555">'right'</span>], max_depth, min_size, depth+<span style="color: #B452CD">1</span>)
|
||||
|
||||
<span style="color: #228B22"># Build a decision tree</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">build_tree</span>(train, max_depth, min_size):
|
||||
root = get_split(train)
|
||||
split(root, max_depth, min_size, <span style="color: #B452CD">1</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> root
|
||||
|
||||
<span style="color: #228B22"># Make a prediction with a decision tree</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">predict</span>(node, row):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> row[node[<span style="color: #CD5555">'index'</span>]] < node[<span style="color: #CD5555">'value'</span>]:
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #658b00">isinstance</span>(node[<span style="color: #CD5555">'left'</span>], <span style="color: #658b00">dict</span>):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> predict(node[<span style="color: #CD5555">'left'</span>], row)
|
||||
<span style="color: #8B008B; font-weight: bold">else</span>:
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> node[<span style="color: #CD5555">'left'</span>]
|
||||
<span style="color: #8B008B; font-weight: bold">else</span>:
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #658b00">isinstance</span>(node[<span style="color: #CD5555">'right'</span>], <span style="color: #658b00">dict</span>):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> predict(node[<span style="color: #CD5555">'right'</span>], row)
|
||||
<span style="color: #8B008B; font-weight: bold">else</span>:
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> node[<span style="color: #CD5555">'right'</span>]
|
||||
|
||||
<span style="color: #228B22"># Classification and Regression Tree Algorithm</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">decision_tree</span>(train, test, max_depth, min_size):
|
||||
tree = build_tree(train, max_depth, min_size)
|
||||
predictions = <span style="color: #658b00">list</span>()
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> row <span style="color: #8B008B">in</span> test:
|
||||
prediction = predict(tree, row)
|
||||
predictions.append(prediction)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span>(predictions)
|
||||
|
||||
<span style="color: #228B22"># Test CART </span>
|
||||
seed(<span style="color: #B452CD">1</span>)
|
||||
<span style="color: #228B22"># load and prepare data</span>
|
||||
filename = <span style="color: #CD5555">'DataFiles/rideclass.csv'</span>
|
||||
dataset = load_csv(filename)
|
||||
<span style="color: #228B22"># convert string attributes to integers</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">len</span>(dataset[<span style="color: #B452CD">0</span>])):
|
||||
str_column_to_float(dataset, i)
|
||||
<span style="color: #228B22"># evaluate algorithm</span>
|
||||
n_folds = <span style="color: #B452CD">5</span>
|
||||
max_depth = <span style="color: #B452CD">5</span>
|
||||
min_size = <span style="color: #B452CD">10</span>
|
||||
scores = evaluate_algorithm(dataset, decision_tree, n_folds, max_depth, min_size)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">'Scores: %s'</span> % scores)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">'Mean Accuracy: %.3f%%'</span> % (<span style="color: #658b00">sum</span>(scores)/<span style="color: #658b00">float</span>(<span style="color: #658b00">len</span>(scores))))
|
||||
dataset = [[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>],
|
||||
[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">2</span>,<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">2</span>,<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>],
|
||||
[<span style="color: #B452CD">1</span>,<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>],
|
||||
[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>],
|
||||
[<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>]]
|
||||
|
||||
split = get_split(dataset)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">'Split: [X%d < %.3f]'</span> % ((split[<span style="color: #CD5555">'index'</span>]+<span style="color: #B452CD">1</span>), split[<span style="color: #CD5555">'value'</span>]))
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec16">Entropy and the ID3 algorithm </h2>
|
||||
<h2 id="___sec20">Entropy and the ID3 algorithm </h2>
|
||||
|
||||
<p>
|
||||
ID3, learns decision trees by constructing
|
||||
@@ -825,15 +897,216 @@ attributes at each step while growing the tree.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec17">Implementing the ID3 Algorithm </h2>
|
||||
<h2 id="___sec21">Implementing the ID3 Algorithm </h2>
|
||||
|
||||
<p>
|
||||
<b>more text to come here</b>, material presented during lecture Friday Oct 25.
|
||||
import re
|
||||
import math
|
||||
from collections import deque
|
||||
|
||||
<p>
|
||||
<!-- x is examples in training set -->
|
||||
<!-- y is set of attributes -->
|
||||
<!-- label is target attributes -->
|
||||
<!-- Node is a class which has properties values, childs, and next -->
|
||||
<!-- root is top node in the decision tree -->
|
||||
|
||||
<p>
|
||||
class Node(object):
|
||||
def __init__(self):
|
||||
self.value = None
|
||||
self.next = None
|
||||
self.childs = None
|
||||
|
||||
<p>
|
||||
<!-- Simple class of Decision Tree -->
|
||||
<!-- Aimed for who want to learn Decision Tree, so it is not optimized -->
|
||||
class DecisionTree(object):
|
||||
def __init__(self, sample, attributes, labels):
|
||||
self.sample = sample
|
||||
self.attributes = attributes
|
||||
self.labels = labels
|
||||
self.labelCodes = None
|
||||
self.labelCodesCount = None
|
||||
self.initLabelCodes()
|
||||
# print(self.labelCodes)
|
||||
self.root = None
|
||||
self.entropy = self.getEntropy([x for x in range(len(self.labels))])
|
||||
|
||||
<p>
|
||||
def initLabelCodes(self):
|
||||
self.labelCodes = []
|
||||
self.labelCodesCount = []
|
||||
for l in self.labels:
|
||||
if l not in self.labelCodes:
|
||||
self.labelCodes.append(l)
|
||||
self.labelCodesCount.append(0)
|
||||
self.labelCodesCount[self.labelCodes.index(l)] += 1
|
||||
|
||||
<p>
|
||||
def getLabelCodeId(self, sampleId):
|
||||
return self.labelCodes.index(self.labels[sampleId])
|
||||
|
||||
<p>
|
||||
def getAttributeValues(self, sampleIds, attributeId):
|
||||
vals = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in vals:
|
||||
vals.append(val)
|
||||
# print(vals)
|
||||
return vals
|
||||
|
||||
<p>
|
||||
def getEntropy(self, sampleIds):
|
||||
entropy = 0
|
||||
labelCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCount[self.getLabelCodeId(sid)] += 1
|
||||
# print("-ge", labelCount)
|
||||
for lv in labelCount:
|
||||
# print(lv)
|
||||
if lv != 0:
|
||||
entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
|
||||
else:
|
||||
entropy += 0
|
||||
return entropy
|
||||
|
||||
<p>
|
||||
def getDominantLabel(self, sampleIds):
|
||||
labelCodesCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
|
||||
return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
|
||||
|
||||
<p>
|
||||
def getInformationGain(self, sampleIds, attributeId):
|
||||
gain = self.getEntropy(sampleIds)
|
||||
attributeVals = []
|
||||
attributeValsCount = []
|
||||
attributeValsIds = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in attributeVals:
|
||||
attributeVals.append(val)
|
||||
attributeValsCount.append(0)
|
||||
attributeValsIds.append([])
|
||||
vid = attributeVals.index(val)
|
||||
attributeValsCount[vid] += 1
|
||||
attributeValsIds[vid].append(sid)
|
||||
# print("-gig", self.attributes[attributeId])
|
||||
for vc, vids in zip(attributeValsCount, attributeValsIds):
|
||||
# print("-gig", vids)
|
||||
gain -= vc/len(sampleIds) * self.getEntropy(vids)
|
||||
return gain
|
||||
|
||||
<p>
|
||||
def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
|
||||
attributesEntropy = [0] * len(attributeIds)
|
||||
for i, attId in zip(range(len(attributeIds)), attributeIds):
|
||||
attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
|
||||
maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
|
||||
return self.attributes[maxId], maxId
|
||||
|
||||
<p>
|
||||
def isSingleLabeled(self, sampleIds):
|
||||
label = self.labels[sampleIds[0]]
|
||||
for sid in sampleIds:
|
||||
if self.labels[sid] != label:
|
||||
return False
|
||||
return True
|
||||
|
||||
<p>
|
||||
def getLabel(self, sampleId):
|
||||
return self.labels[sampleId]
|
||||
|
||||
<p>
|
||||
def id3(self):
|
||||
sampleIds = [x for x in range(len(self.sample))]
|
||||
attributeIds = [x for x in range(len(self.attributes))]
|
||||
self.root = self.id3Recv(sampleIds, attributeIds, self.root)
|
||||
|
||||
<p>
|
||||
def id3Recv(self, sampleIds, attributeIds, root):
|
||||
root = Node() # Initialize current root
|
||||
if self.isSingleLabeled(sampleIds):
|
||||
root.value = self.labels[sampleIds[0]]
|
||||
return root
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) == 0:
|
||||
root.value = self.getDominantLabel(sampleIds)
|
||||
return root
|
||||
bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
|
||||
sampleIds, attributeIds)
|
||||
# print(bestAttrName)
|
||||
root.value = bestAttrName
|
||||
root.childs = [] # Create list of children
|
||||
for value in self.getAttributeValues(sampleIds, bestAttrId):
|
||||
# print(value)
|
||||
child = Node()
|
||||
child.value = value
|
||||
root.childs.append(child) # Append new child node to current
|
||||
# root
|
||||
childSampleIds = []
|
||||
for sid in sampleIds:
|
||||
if self.sample[sid][bestAttrId] == value:
|
||||
childSampleIds.append(sid)
|
||||
if len(childSampleIds) == 0:
|
||||
child.next = self.getDominantLabel(sampleIds)
|
||||
else:
|
||||
# print(bestAttrName, bestAttrId)
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) > 0 and bestAttrId in attributeIds:
|
||||
toRemove = attributeIds.index(bestAttrId)
|
||||
attributeIds.pop(toRemove)
|
||||
child.next = self.id3Recv(
|
||||
childSampleIds, attributeIds, child.next)
|
||||
return root
|
||||
|
||||
<p>
|
||||
def printTree(self):
|
||||
if self.root:
|
||||
roots = deque()
|
||||
roots.append(self.root)
|
||||
while len(roots) > 0:
|
||||
root = roots.popleft()
|
||||
print(root.value)
|
||||
if root.childs:
|
||||
for child in root.childs:
|
||||
print('({})'.format(child.value))
|
||||
roots.append(child.next)
|
||||
elif root.next:
|
||||
print(root.next)
|
||||
|
||||
<p>
|
||||
def test():
|
||||
f = open('DataFiles/rideclass.csv')
|
||||
attributes = f.readline().split(',')
|
||||
attributes = attributes[1:len(attributes)-1]
|
||||
print(attributes)
|
||||
sample = f.readlines()
|
||||
f.close()
|
||||
for i in range(len(sample)):
|
||||
sample[i] = re.sub('\d+,', '', sample[i])
|
||||
sample[i] = sample[i].strip().split(',')
|
||||
labels = []
|
||||
for s in sample:
|
||||
labels.append(s.pop())
|
||||
# print(sample)
|
||||
# print(labels)
|
||||
decisionTree = DecisionTree(sample, attributes, labels)
|
||||
print("System entropy {}".format(decisionTree.entropy))
|
||||
decisionTree.id3()
|
||||
decisionTree.printTree()
|
||||
|
||||
<p>
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec18">Cancer Data again now with Decision Trees </h2>
|
||||
<h2 id="___sec22">Cancer Data again now with Decision Trees and other Methods </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -882,7 +1155,7 @@ deep_tree_clf.fit(X_train_scaled, y_train)
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec19">Another example, the moons again </h2>
|
||||
<h2 id="___sec23">Another example, the moons again </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -954,7 +1227,7 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec20">Playing around with regions </h2>
|
||||
<h2 id="___sec24">Playing around with regions </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -962,7 +1235,7 @@ plt.show()
|
||||
Xs = np.random.rand(<span style="color: #B452CD">100</span>, <span style="color: #B452CD">2</span>) - <span style="color: #B452CD">0.5</span>
|
||||
ys = (Xs[:, <span style="color: #B452CD">0</span>] > <span style="color: #B452CD">0</span>).astype(np.float32) * <span style="color: #B452CD">2</span>
|
||||
|
||||
angle = np.pi / <span style="color: #B452CD">4</span>
|
||||
angle = np.pi/<span style="color: #B452CD">4</span>
|
||||
rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
|
||||
Xsr = Xs.dot(rotation_matrix)
|
||||
|
||||
@@ -982,7 +1255,7 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec21">Regression trees </h2>
|
||||
<h2 id="___sec25">Regression trees </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -1004,7 +1277,7 @@ tree_reg.fit(X, y)
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec22">Final regressor code </h2>
|
||||
<h2 id="___sec26">Final regressor code </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -1082,7 +1355,7 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec23">Pros and cons of trees, pros </h2>
|
||||
<h2 id="___sec27">Pros and cons of trees, pros </h2>
|
||||
|
||||
<ul>
|
||||
<li> White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines)</li>
|
||||
@@ -1096,7 +1369,7 @@ plt.show()
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec24">Disadvantages </h2>
|
||||
<h2 id="___sec28">Disadvantages </h2>
|
||||
|
||||
<ul>
|
||||
<li> Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches</li>
|
||||
@@ -1113,7 +1386,7 @@ However, by aggregating many decision trees, using methods like bagging, random
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec25">Bagging </h2>
|
||||
<h2 id="___sec29">Bagging </h2>
|
||||
|
||||
<p>
|
||||
The <b>plain</b> decision trees suffer from high
|
||||
@@ -1129,6 +1402,11 @@ of \( n \) to \( p \) is moderately large.
|
||||
general-purpose procedure for reducing the variance of a statistical
|
||||
learning method.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec30">More bagging </h2>
|
||||
|
||||
<p>
|
||||
Bagging typically results in improved accuracy
|
||||
over prediction using a single tree. Unfortunately, however, it can be
|
||||
@@ -1156,7 +1434,7 @@ predictor, averaged over all \( B \) trees.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec26">Simple example, head or tail </h2>
|
||||
<h2 id="___sec31">Simple example, head or tail </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -1176,7 +1454,58 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec27">Random forests </h2>
|
||||
<h2 id="___sec32">Bagging Example </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_moons
|
||||
|
||||
X, y = make_moons(n_samples=<span style="color: #B452CD">500</span>, noise=<span style="color: #B452CD">0.30</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> VotingClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||||
|
||||
log_clf = LogisticRegression(solver=<span style="color: #CD5555">"liblinear"</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf = RandomForestClassifier(n_estimators=<span style="color: #B452CD">10</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
svm_clf = SVC(gamma=<span style="color: #CD5555">"auto"</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[(<span style="color: #CD5555">'lr'</span>, log_clf), (<span style="color: #CD5555">'rf'</span>, rnd_clf), (<span style="color: #CD5555">'svc'</span>, svm_clf)],
|
||||
voting=<span style="color: #CD5555">'hard'</span>)
|
||||
|
||||
voting_clf.fit(X_train, y_train)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> clf <span style="color: #8B008B">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(clf.<span style="color: #00688B">__class__</span>.<span style="color: #00688B">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
|
||||
log_clf = LogisticRegression(solver=<span style="color: #CD5555">"liblinear"</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf = RandomForestClassifier(n_estimators=<span style="color: #B452CD">10</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
svm_clf = SVC(gamma=<span style="color: #CD5555">"auto"</span>, probability=<span style="color: #658b00">True</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[(<span style="color: #CD5555">'lr'</span>, log_clf), (<span style="color: #CD5555">'rf'</span>, rnd_clf), (<span style="color: #CD5555">'svc'</span>, svm_clf)],
|
||||
voting=<span style="color: #CD5555">'soft'</span>)
|
||||
voting_clf.fit(X_train, y_train)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> clf <span style="color: #8B008B">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(clf.<span style="color: #00688B">__class__</span>.<span style="color: #00688B">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec33">Random forests </h2>
|
||||
|
||||
<p>
|
||||
Random forests provide an improvement over bagged trees by way of a
|
||||
@@ -1220,7 +1549,7 @@ this setting.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec28">A simple scikit-learn example </h2>
|
||||
<h2 id="___sec34">A simple scikit-learn example </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -1238,7 +1567,7 @@ accuracy = cross_validate(Random_Forest_model,X,Y,cv=<span style="color: #B452CD
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec29">Please, not the moons again! </h2>
|
||||
<h2 id="___sec35">Please, not the moons again! </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -1296,7 +1625,7 @@ voting_clf.fit(X_train, y_train)
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec30">Bagging examples </h2>
|
||||
<h2 id="___sec36">Bagging examples </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -1357,7 +1686,7 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec31">Then random forests </h2>
|
||||
<h2 id="___sec37">Then random forests </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -1376,12 +1705,6 @@ rnd_clf.fit(X_train, y_train)
|
||||
y_pred_rf = rnd_clf.predict(X_test)
|
||||
np.sum(y_pred == y_pred_rf) / <span style="color: #658b00">len</span>(y_pred)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec32">Boosting and more </h2>
|
||||
More material to come here.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -92,30 +92,32 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
('A Classification Tree', 2, None, '___sec12'),
|
||||
('Growing a classification tree', 2, None, '___sec13'),
|
||||
('Classification tree, how to split nodes', 2, None, '___sec14'),
|
||||
('The CART (Classification and Regression Tree) algorithm',
|
||||
('Visualizing the Tree, Classification', 2, None, '___sec15'),
|
||||
('Visualizing the Tree, The Moons', 2, None, '___sec16'),
|
||||
('Computing the Gini index', 2, None, '___sec17'),
|
||||
('Simple Python Code to read in Data', 2, None, '___sec18'),
|
||||
('Computing the Gini Factor', 2, None, '___sec19'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec20'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec21'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec15'),
|
||||
('Entropy and the ID3 algorithm', 2, None, '___sec16'),
|
||||
('Implementing the ID3 Algorithm', 2, None, '___sec17'),
|
||||
('Cancer Data again now with Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Another example, the moons again', 2, None, '___sec19'),
|
||||
('Playing around with regions', 2, None, '___sec20'),
|
||||
('Regression trees', 2, None, '___sec21'),
|
||||
('Final regressor code', 2, None, '___sec22'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec23'),
|
||||
('Disadvantages', 2, None, '___sec24'),
|
||||
('Bagging', 2, None, '___sec25'),
|
||||
('Simple example, head or tail', 2, None, '___sec26'),
|
||||
('Random forests', 2, None, '___sec27'),
|
||||
('A simple scikit-learn example', 2, None, '___sec28'),
|
||||
('Please, not the moons again!', 2, None, '___sec29'),
|
||||
('Bagging examples', 2, None, '___sec30'),
|
||||
('Then random forests', 2, None, '___sec31'),
|
||||
('Boosting and more', 2, None, '___sec32')]}
|
||||
'___sec22'),
|
||||
('Another example, the moons again', 2, None, '___sec23'),
|
||||
('Playing around with regions', 2, None, '___sec24'),
|
||||
('Regression trees', 2, None, '___sec25'),
|
||||
('Final regressor code', 2, None, '___sec26'),
|
||||
('Pros and cons of trees, pros', 2, None, '___sec27'),
|
||||
('Disadvantages', 2, None, '___sec28'),
|
||||
('Bagging', 2, None, '___sec29'),
|
||||
('More bagging', 2, None, '___sec30'),
|
||||
('Simple example, head or tail', 2, None, '___sec31'),
|
||||
('Bagging Example', 2, None, '___sec32'),
|
||||
('Random forests', 2, None, '___sec33'),
|
||||
('A simple scikit-learn example', 2, None, '___sec34'),
|
||||
('Please, not the moons again!', 2, None, '___sec35'),
|
||||
('Bagging examples', 2, None, '___sec36'),
|
||||
('Then random forests', 2, None, '___sec37')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -157,7 +159,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Oct 29, 2019</h4></center> <!-- date -->
|
||||
<center><h4>Oct 31, 2019</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
@@ -611,71 +613,196 @@ $$
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec15">The CART (Classification and Regression Tree) algorithm </h2>
|
||||
<h2 id="___sec15">Visualizing the Tree, Classification </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> confusion_matrix
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> export_graphviz
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> Image
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">pydot</span> <span style="color: #008000; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
|
||||
|
||||
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
||||
X <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>DataFrame(cancer<span style="color: #666666">.</span>data, columns<span style="color: #666666">=</span>cancer<span style="color: #666666">.</span>feature_names)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(X)
|
||||
y <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>Categorical<span style="color: #666666">.</span>from_codes(cancer<span style="color: #666666">.</span>target, cancer<span style="color: #666666">.</span>target_names)
|
||||
y <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>get_dummies(y)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(y)
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(X, y, random_state<span style="color: #666666">=1</span>)
|
||||
tree_clf <span style="color: #666666">=</span> DecisionTreeClassifier(max_depth<span style="color: #666666">=5</span>)
|
||||
tree_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file<span style="color: #666666">=</span><span style="color: #BA2121">"DataFiles/cancer.dot"</span>,
|
||||
feature_names<span style="color: #666666">=</span>cancer<span style="color: #666666">.</span>feature_names,
|
||||
class_names<span style="color: #666666">=</span>cancer<span style="color: #666666">.</span>target_names,
|
||||
rounded<span style="color: #666666">=</span><span style="color: #008000">True</span>,
|
||||
filled<span style="color: #666666">=</span><span style="color: #008000">True</span>
|
||||
)
|
||||
cmd <span style="color: #666666">=</span> <span style="color: #BA2121">'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'</span>
|
||||
os<span style="color: #666666">.</span>system(cmd)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec16">Visualizing the Tree, The Moons </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Common imports</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_moons
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> export_graphviz
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">pydot</span> <span style="color: #008000; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
||||
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">42</span>)
|
||||
X, y <span style="color: #666666">=</span> make_moons(n_samples<span style="color: #666666">=100</span>, noise<span style="color: #666666">=0.25</span>, random_state<span style="color: #666666">=53</span>)
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(X,y,random_state<span style="color: #666666">=0</span>)
|
||||
tree_clf <span style="color: #666666">=</span> DecisionTreeClassifier(max_depth<span style="color: #666666">=5</span>)
|
||||
tree_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file<span style="color: #666666">=</span><span style="color: #BA2121">"DataFiles/moons.dot"</span>,
|
||||
rounded<span style="color: #666666">=</span><span style="color: #008000">True</span>,
|
||||
filled<span style="color: #666666">=</span><span style="color: #008000">True</span>
|
||||
)
|
||||
cmd <span style="color: #666666">=</span> <span style="color: #BA2121">'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png'</span>
|
||||
os<span style="color: #666666">.</span>system(cmd)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec17">Computing the Gini index </h2>
|
||||
|
||||
<p>
|
||||
The above functions (gini, entropy and misclassification error) are important components of the so-called CART algorithm. We will discuss this algorithm first before we move on to the information gain algorithm ID3.
|
||||
The example we will look at is a classical one in many Machine
|
||||
Learning applications. Based on various meteorological features, we
|
||||
have several so-called attributes which decide whether we at the end
|
||||
will do some outdoor activity like skiing, going for a bike ride etc
|
||||
etc. The table here contains the feautures <b>outlook</b>, <b>temperature</b>,
|
||||
<b>humidity</b> and <b>wind</b>. The target or output is whether we ride
|
||||
(True=1) or whether we do something else that day (False=0). The
|
||||
attributes for each feature are then sunny, overcast and rain for the
|
||||
outlook, hot, cold and mild for temperature, high and normal for
|
||||
humidity and weak and strong for wind.
|
||||
|
||||
<p>
|
||||
The table here summarizes the various attributes and
|
||||
<table border="1">
|
||||
<thead>
|
||||
<tr><th align="center">Day</th> <th align="center">Outlook </th> <th align="center">Temperature</th> <th align="center">Humidity</th> <th align="center"> Wind </th> <th align="center">Ride</th> </tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td align="center"> 1 </td> <td align="center"> Sunny </td> <td align="center"> Hot </td> <td align="center"> High </td> <td align="center"> Weak </td> <td align="center"> 0 </td> </tr>
|
||||
<tr><td align="center"> 2 </td> <td align="center"> Sunny </td> <td align="center"> Hot </td> <td align="center"> High </td> <td align="center"> Strong </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 3 </td> <td align="center"> Overcast </td> <td align="center"> Hot </td> <td align="center"> High </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 4 </td> <td align="center"> Rain </td> <td align="center"> Mild </td> <td align="center"> High </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 5 </td> <td align="center"> Rain </td> <td align="center"> Cool </td> <td align="center"> Normal </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 6 </td> <td align="center"> Rain </td> <td align="center"> Cool </td> <td align="center"> Normal </td> <td align="center"> Strong </td> <td align="center"> 0 </td> </tr>
|
||||
<tr><td align="center"> 7 </td> <td align="center"> Overcast </td> <td align="center"> Cool </td> <td align="center"> Normal </td> <td align="center"> Strong </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 8 </td> <td align="center"> Sunny </td> <td align="center"> Mild </td> <td align="center"> High </td> <td align="center"> Weak </td> <td align="center"> 0 </td> </tr>
|
||||
<tr><td align="center"> 9 </td> <td align="center"> Sunny </td> <td align="center"> Cool </td> <td align="center"> Normal </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 10 </td> <td align="center"> Rain </td> <td align="center"> Mild </td> <td align="center"> Normal </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 11 </td> <td align="center"> Sunny </td> <td align="center"> Mild </td> <td align="center"> Normal </td> <td align="center"> Strong </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 12 </td> <td align="center"> Overcast </td> <td align="center"> Mild </td> <td align="center"> High </td> <td align="center"> Strong </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 13 </td> <td align="center"> Overcast </td> <td align="center"> Hot </td> <td align="center"> Normal </td> <td align="center"> Weak </td> <td align="center"> 1 </td> </tr>
|
||||
<tr><td align="center"> 14 </td> <td align="center"> Rain </td> <td align="center"> Mild </td> <td align="center"> High </td> <td align="center"> Strong </td> <td align="center"> 0 </td> </tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec18">Simple Python Code to read in Data </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> seed
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> randrange
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">csv</span> <span style="color: #008000; font-weight: bold">import</span> reader
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Load a CSV file</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">load_csv</span>(filename):
|
||||
<span style="color: #008000">file</span> <span style="color: #666666">=</span> <span style="color: #008000">open</span>(filename, <span style="color: #BA2121">"rb"</span>)
|
||||
lines <span style="color: #666666">=</span> reader(<span style="color: #008000">file</span>)
|
||||
dataset <span style="color: #666666">=</span> <span style="color: #008000">list</span>(lines)
|
||||
<span style="color: #008000; font-weight: bold">return</span> dataset
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Convert string column to float</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">str_column_to_float</span>(dataset, column):
|
||||
<span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> dataset:
|
||||
row[column] <span style="color: #666666">=</span> <span style="color: #008000">float</span>(row[column]<span style="color: #666666">.</span>strip())
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Split a dataset into k folds</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">cross_validation_split</span>(dataset, n_folds):
|
||||
dataset_split <span style="color: #666666">=</span> <span style="color: #008000">list</span>()
|
||||
dataset_copy <span style="color: #666666">=</span> <span style="color: #008000">list</span>(dataset)
|
||||
fold_size <span style="color: #666666">=</span> <span style="color: #008000">int</span>(<span style="color: #008000">len</span>(dataset) <span style="color: #666666">/</span> n_folds)
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_folds):
|
||||
fold <span style="color: #666666">=</span> <span style="color: #008000">list</span>()
|
||||
<span style="color: #008000; font-weight: bold">while</span> <span style="color: #008000">len</span>(fold) <span style="color: #666666"><</span> fold_size:
|
||||
index <span style="color: #666666">=</span> randrange(<span style="color: #008000">len</span>(dataset_copy))
|
||||
fold<span style="color: #666666">.</span>append(dataset_copy<span style="color: #666666">.</span>pop(index))
|
||||
dataset_split<span style="color: #666666">.</span>append(fold)
|
||||
<span style="color: #008000; font-weight: bold">return</span> dataset_split
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Calculate accuracy percentage</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">accuracy_metric</span>(actual, predicted):
|
||||
correct <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(actual)):
|
||||
<span style="color: #008000; font-weight: bold">if</span> actual[i] <span style="color: #666666">==</span> predicted[i]:
|
||||
correct <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
||||
<span style="color: #008000; font-weight: bold">return</span> correct <span style="color: #666666">/</span> <span style="color: #008000">float</span>(<span style="color: #008000">len</span>(actual)) <span style="color: #666666">*</span> <span style="color: #666666">100.0</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Evaluate an algorithm using a cross validation split</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">evaluate_algorithm</span>(dataset, algorithm, n_folds, <span style="color: #666666">*</span>args):
|
||||
folds <span style="color: #666666">=</span> cross_validation_split(dataset, n_folds)
|
||||
scores <span style="color: #666666">=</span> <span style="color: #008000">list</span>()
|
||||
<span style="color: #008000; font-weight: bold">for</span> fold <span style="color: #AA22FF; font-weight: bold">in</span> folds:
|
||||
train_set <span style="color: #666666">=</span> <span style="color: #008000">list</span>(folds)
|
||||
train_set<span style="color: #666666">.</span>remove(fold)
|
||||
train_set <span style="color: #666666">=</span> <span style="color: #008000">sum</span>(train_set, [])
|
||||
test_set <span style="color: #666666">=</span> <span style="color: #008000">list</span>()
|
||||
<span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> fold:
|
||||
row_copy <span style="color: #666666">=</span> <span style="color: #008000">list</span>(row)
|
||||
test_set<span style="color: #666666">.</span>append(row_copy)
|
||||
row_copy[<span style="color: #666666">-1</span>] <span style="color: #666666">=</span> <span style="color: #008000">None</span>
|
||||
predicted <span style="color: #666666">=</span> algorithm(train_set, test_set, <span style="color: #666666">*</span>args)
|
||||
actual <span style="color: #666666">=</span> [row[<span style="color: #666666">-1</span>] <span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> fold]
|
||||
accuracy <span style="color: #666666">=</span> accuracy_metric(actual, predicted)
|
||||
scores<span style="color: #666666">.</span>append(accuracy)
|
||||
<span style="color: #008000; font-weight: bold">return</span> scores
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Split a dataset based on an attribute and an attribute value</span>
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Common imports</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler, OneHotEncoder
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.compose</span> <span style="color: #008000; font-weight: bold">import</span> ColumnTransformer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> Image
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">pydot</span> <span style="color: #008000; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Where to save the figures and data files</span>
|
||||
PROJECT_ROOT_DIR <span style="color: #666666">=</span> <span style="color: #BA2121">"Results"</span>
|
||||
FIGURE_ID <span style="color: #666666">=</span> <span style="color: #BA2121">"Results/FigureFiles"</span>
|
||||
DATA_ID <span style="color: #666666">=</span> <span style="color: #BA2121">"DataFiles/"</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(PROJECT_ROOT_DIR):
|
||||
os<span style="color: #666666">.</span>mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(FIGURE_ID):
|
||||
os<span style="color: #666666">.</span>makedirs(FIGURE_ID)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(DATA_ID):
|
||||
os<span style="color: #666666">.</span>makedirs(DATA_ID)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">image_path</span>(fig_id):
|
||||
<span style="color: #008000; font-weight: bold">return</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(FIGURE_ID, fig_id)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">data_path</span>(dat_id):
|
||||
<span style="color: #008000; font-weight: bold">return</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(DATA_ID, dat_id)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">save_fig</span>(fig_id):
|
||||
plt<span style="color: #666666">.</span>savefig(image_path(fig_id) <span style="color: #666666">+</span> <span style="color: #BA2121">".png"</span>, format<span style="color: #666666">=</span><span style="color: #BA2121">'png'</span>)
|
||||
|
||||
infile <span style="color: #666666">=</span> <span style="color: #008000">open</span>(data_path(<span style="color: #BA2121">"ride.csv"</span>),<span style="color: #BA2121">'r'</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Read the experimental data with Pandas</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> display
|
||||
ridedata <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>read_csv(infile,names <span style="color: #666666">=</span> (<span style="color: #BA2121">'Outlook'</span>,<span style="color: #BA2121">'Temperature'</span>,<span style="color: #BA2121">'Humidity'</span>,<span style="color: #BA2121">'Wind'</span>,<span style="color: #BA2121">'Ride'</span>))
|
||||
ridedata <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>DataFrame(ridedata)
|
||||
display(ridedata)
|
||||
<span style="color: #408080; font-style: italic"># Features and targets</span>
|
||||
X <span style="color: #666666">=</span> ridedata<span style="color: #666666">.</span>loc[:, ridedata<span style="color: #666666">.</span>columns <span style="color: #666666">!=</span> <span style="color: #BA2121">'Ride'</span>]<span style="color: #666666">.</span>values
|
||||
display(X)
|
||||
y <span style="color: #666666">=</span> ridedata<span style="color: #666666">.</span>loc[:, ridedata<span style="color: #666666">.</span>columns <span style="color: #666666">==</span> <span style="color: #BA2121">'Ride'</span>]<span style="color: #666666">.</span>values
|
||||
display(y)
|
||||
<span style="color: #408080; font-style: italic"># Categorical variables to one-hot's</span>
|
||||
onehotencoder <span style="color: #666666">=</span> OneHotEncoder(categories<span style="color: #666666">=</span><span style="color: #BA2121">"auto"</span>)
|
||||
|
||||
X <span style="color: #666666">=</span> ColumnTransformer([(<span style="color: #BA2121">""</span>, onehotencoder)])<span style="color: #666666">.</span>fit_transform(X)
|
||||
y<span style="color: #666666">.</span>shape
|
||||
|
||||
display(X)
|
||||
display(y)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec19">Computing the Gini Factor </h2>
|
||||
|
||||
<p>
|
||||
The above functions (gini, entropy and misclassification error) are
|
||||
important components of the so-called CART algorithm. We will discuss
|
||||
this algorithm below after we have discussed the information gain
|
||||
algorithm ID3.
|
||||
|
||||
<p>
|
||||
In the example here we have converted all our attributes into numerical values \( 0,1,2 \) etc.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Split a dataset based on an attribute and an attribute value</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">test_split</span>(index, value, dataset):
|
||||
left, right <span style="color: #666666">=</span> <span style="color: #008000">list</span>(), <span style="color: #008000">list</span>()
|
||||
<span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> dataset:
|
||||
@@ -704,7 +831,7 @@ The above functions (gini, entropy and misclassification error) are important co
|
||||
<span style="color: #408080; font-style: italic"># weight the group score by its relative size</span>
|
||||
gini <span style="color: #666666">+=</span> (<span style="color: #666666">1.0</span> <span style="color: #666666">-</span> score) <span style="color: #666666">*</span> (size <span style="color: #666666">/</span> n_instances)
|
||||
<span style="color: #008000; font-weight: bold">return</span> gini
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Select the best split point for a dataset</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">get_split</span>(dataset):
|
||||
class_values <span style="color: #666666">=</span> <span style="color: #008000">list</span>(<span style="color: #008000">set</span>(row[<span style="color: #666666">-1</span>] <span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> dataset))
|
||||
@@ -713,88 +840,33 @@ The above functions (gini, entropy and misclassification error) are important co
|
||||
<span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> dataset:
|
||||
groups <span style="color: #666666">=</span> test_split(index, row[index], dataset)
|
||||
gini <span style="color: #666666">=</span> gini_index(groups, class_values)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'X</span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121"> < </span><span style="color: #BB6688; font-weight: bold">%.3f</span><span style="color: #BA2121"> Gini=</span><span style="color: #BB6688; font-weight: bold">%.3f</span><span style="color: #BA2121">'</span> <span style="color: #666666">%</span> ((index<span style="color: #666666">+1</span>), row[index], gini))
|
||||
<span style="color: #008000; font-weight: bold">if</span> gini <span style="color: #666666"><</span> b_score:
|
||||
b_index, b_value, b_score, b_groups <span style="color: #666666">=</span> index, row[index], gini, groups
|
||||
<span style="color: #008000; font-weight: bold">return</span> {<span style="color: #BA2121">'index'</span>:b_index, <span style="color: #BA2121">'value'</span>:b_value, <span style="color: #BA2121">'groups'</span>:b_groups}
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Create a terminal node value</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">to_terminal</span>(group):
|
||||
outcomes <span style="color: #666666">=</span> [row[<span style="color: #666666">-1</span>] <span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> group]
|
||||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">max</span>(<span style="color: #008000">set</span>(outcomes), key<span style="color: #666666">=</span>outcomes<span style="color: #666666">.</span>count)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Create child splits for a node or make terminal</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">split</span>(node, max_depth, min_size, depth):
|
||||
left, right <span style="color: #666666">=</span> node[<span style="color: #BA2121">'groups'</span>]
|
||||
<span style="color: #008000; font-weight: bold">del</span>(node[<span style="color: #BA2121">'groups'</span>])
|
||||
<span style="color: #408080; font-style: italic"># check for a no split</span>
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> left <span style="color: #AA22FF; font-weight: bold">or</span> <span style="color: #AA22FF; font-weight: bold">not</span> right:
|
||||
node[<span style="color: #BA2121">'left'</span>] <span style="color: #666666">=</span> node[<span style="color: #BA2121">'right'</span>] <span style="color: #666666">=</span> to_terminal(left <span style="color: #666666">+</span> right)
|
||||
<span style="color: #008000; font-weight: bold">return</span>
|
||||
<span style="color: #408080; font-style: italic"># check for max depth</span>
|
||||
<span style="color: #008000; font-weight: bold">if</span> depth <span style="color: #666666">>=</span> max_depth:
|
||||
node[<span style="color: #BA2121">'left'</span>], node[<span style="color: #BA2121">'right'</span>] <span style="color: #666666">=</span> to_terminal(left), to_terminal(right)
|
||||
<span style="color: #008000; font-weight: bold">return</span>
|
||||
<span style="color: #408080; font-style: italic"># process left child</span>
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">len</span>(left) <span style="color: #666666"><=</span> min_size:
|
||||
node[<span style="color: #BA2121">'left'</span>] <span style="color: #666666">=</span> to_terminal(left)
|
||||
<span style="color: #008000; font-weight: bold">else</span>:
|
||||
node[<span style="color: #BA2121">'left'</span>] <span style="color: #666666">=</span> get_split(left)
|
||||
split(node[<span style="color: #BA2121">'left'</span>], max_depth, min_size, depth<span style="color: #666666">+1</span>)
|
||||
<span style="color: #408080; font-style: italic"># process right child</span>
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">len</span>(right) <span style="color: #666666"><=</span> min_size:
|
||||
node[<span style="color: #BA2121">'right'</span>] <span style="color: #666666">=</span> to_terminal(right)
|
||||
<span style="color: #008000; font-weight: bold">else</span>:
|
||||
node[<span style="color: #BA2121">'right'</span>] <span style="color: #666666">=</span> get_split(right)
|
||||
split(node[<span style="color: #BA2121">'right'</span>], max_depth, min_size, depth<span style="color: #666666">+1</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Build a decision tree</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">build_tree</span>(train, max_depth, min_size):
|
||||
root <span style="color: #666666">=</span> get_split(train)
|
||||
split(root, max_depth, min_size, <span style="color: #666666">1</span>)
|
||||
<span style="color: #008000; font-weight: bold">return</span> root
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Make a prediction with a decision tree</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">predict</span>(node, row):
|
||||
<span style="color: #008000; font-weight: bold">if</span> row[node[<span style="color: #BA2121">'index'</span>]] <span style="color: #666666"><</span> node[<span style="color: #BA2121">'value'</span>]:
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">isinstance</span>(node[<span style="color: #BA2121">'left'</span>], <span style="color: #008000">dict</span>):
|
||||
<span style="color: #008000; font-weight: bold">return</span> predict(node[<span style="color: #BA2121">'left'</span>], row)
|
||||
<span style="color: #008000; font-weight: bold">else</span>:
|
||||
<span style="color: #008000; font-weight: bold">return</span> node[<span style="color: #BA2121">'left'</span>]
|
||||
<span style="color: #008000; font-weight: bold">else</span>:
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">isinstance</span>(node[<span style="color: #BA2121">'right'</span>], <span style="color: #008000">dict</span>):
|
||||
<span style="color: #008000; font-weight: bold">return</span> predict(node[<span style="color: #BA2121">'right'</span>], row)
|
||||
<span style="color: #008000; font-weight: bold">else</span>:
|
||||
<span style="color: #008000; font-weight: bold">return</span> node[<span style="color: #BA2121">'right'</span>]
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Classification and Regression Tree Algorithm</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">decision_tree</span>(train, test, max_depth, min_size):
|
||||
tree <span style="color: #666666">=</span> build_tree(train, max_depth, min_size)
|
||||
predictions <span style="color: #666666">=</span> <span style="color: #008000">list</span>()
|
||||
<span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> test:
|
||||
prediction <span style="color: #666666">=</span> predict(tree, row)
|
||||
predictions<span style="color: #666666">.</span>append(prediction)
|
||||
<span style="color: #008000; font-weight: bold">return</span>(predictions)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Test CART </span>
|
||||
seed(<span style="color: #666666">1</span>)
|
||||
<span style="color: #408080; font-style: italic"># load and prepare data</span>
|
||||
filename <span style="color: #666666">=</span> <span style="color: #BA2121">'DataFiles/rideclass.csv'</span>
|
||||
dataset <span style="color: #666666">=</span> load_csv(filename)
|
||||
<span style="color: #408080; font-style: italic"># convert string attributes to integers</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(dataset[<span style="color: #666666">0</span>])):
|
||||
str_column_to_float(dataset, i)
|
||||
<span style="color: #408080; font-style: italic"># evaluate algorithm</span>
|
||||
n_folds <span style="color: #666666">=</span> <span style="color: #666666">5</span>
|
||||
max_depth <span style="color: #666666">=</span> <span style="color: #666666">5</span>
|
||||
min_size <span style="color: #666666">=</span> <span style="color: #666666">10</span>
|
||||
scores <span style="color: #666666">=</span> evaluate_algorithm(dataset, decision_tree, n_folds, max_depth, min_size)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'Scores: </span><span style="color: #BB6688; font-weight: bold">%s</span><span style="color: #BA2121">'</span> <span style="color: #666666">%</span> scores)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'Mean Accuracy: </span><span style="color: #BB6688; font-weight: bold">%.3f%%</span><span style="color: #BA2121">'</span> <span style="color: #666666">%</span> (<span style="color: #008000">sum</span>(scores)<span style="color: #666666">/</span><span style="color: #008000">float</span>(<span style="color: #008000">len</span>(scores))))
|
||||
dataset <span style="color: #666666">=</span> [[<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>],
|
||||
[<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">2</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">2</span>,<span style="color: #666666">2</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">2</span>,<span style="color: #666666">2</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>],
|
||||
[<span style="color: #666666">1</span>,<span style="color: #666666">2</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">0</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">0</span>],
|
||||
[<span style="color: #666666">0</span>,<span style="color: #666666">2</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">2</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">0</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">1</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>],
|
||||
[<span style="color: #666666">2</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>]]
|
||||
|
||||
split <span style="color: #666666">=</span> get_split(dataset)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'Split: [X</span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121"> < </span><span style="color: #BB6688; font-weight: bold">%.3f</span><span style="color: #BA2121">]'</span> <span style="color: #666666">%</span> ((split[<span style="color: #BA2121">'index'</span>]<span style="color: #666666">+1</span>), split[<span style="color: #BA2121">'value'</span>]))
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec16">Entropy and the ID3 algorithm </h2>
|
||||
<h2 id="___sec20">Entropy and the ID3 algorithm </h2>
|
||||
|
||||
<p>
|
||||
ID3, learns decision trees by constructing
|
||||
@@ -830,15 +902,216 @@ attributes at each step while growing the tree.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec17">Implementing the ID3 Algorithm </h2>
|
||||
<h2 id="___sec21">Implementing the ID3 Algorithm </h2>
|
||||
|
||||
<p>
|
||||
<b>more text to come here</b>, material presented during lecture Friday Oct 25.
|
||||
import re
|
||||
import math
|
||||
from collections import deque
|
||||
|
||||
<p>
|
||||
<!-- x is examples in training set -->
|
||||
<!-- y is set of attributes -->
|
||||
<!-- label is target attributes -->
|
||||
<!-- Node is a class which has properties values, childs, and next -->
|
||||
<!-- root is top node in the decision tree -->
|
||||
|
||||
<p>
|
||||
class Node(object):
|
||||
def __init__(self):
|
||||
self.value = None
|
||||
self.next = None
|
||||
self.childs = None
|
||||
|
||||
<p>
|
||||
<!-- Simple class of Decision Tree -->
|
||||
<!-- Aimed for who want to learn Decision Tree, so it is not optimized -->
|
||||
class DecisionTree(object):
|
||||
def __init__(self, sample, attributes, labels):
|
||||
self.sample = sample
|
||||
self.attributes = attributes
|
||||
self.labels = labels
|
||||
self.labelCodes = None
|
||||
self.labelCodesCount = None
|
||||
self.initLabelCodes()
|
||||
# print(self.labelCodes)
|
||||
self.root = None
|
||||
self.entropy = self.getEntropy([x for x in range(len(self.labels))])
|
||||
|
||||
<p>
|
||||
def initLabelCodes(self):
|
||||
self.labelCodes = []
|
||||
self.labelCodesCount = []
|
||||
for l in self.labels:
|
||||
if l not in self.labelCodes:
|
||||
self.labelCodes.append(l)
|
||||
self.labelCodesCount.append(0)
|
||||
self.labelCodesCount[self.labelCodes.index(l)] += 1
|
||||
|
||||
<p>
|
||||
def getLabelCodeId(self, sampleId):
|
||||
return self.labelCodes.index(self.labels[sampleId])
|
||||
|
||||
<p>
|
||||
def getAttributeValues(self, sampleIds, attributeId):
|
||||
vals = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in vals:
|
||||
vals.append(val)
|
||||
# print(vals)
|
||||
return vals
|
||||
|
||||
<p>
|
||||
def getEntropy(self, sampleIds):
|
||||
entropy = 0
|
||||
labelCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCount[self.getLabelCodeId(sid)] += 1
|
||||
# print("-ge", labelCount)
|
||||
for lv in labelCount:
|
||||
# print(lv)
|
||||
if lv != 0:
|
||||
entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
|
||||
else:
|
||||
entropy += 0
|
||||
return entropy
|
||||
|
||||
<p>
|
||||
def getDominantLabel(self, sampleIds):
|
||||
labelCodesCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
|
||||
return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
|
||||
|
||||
<p>
|
||||
def getInformationGain(self, sampleIds, attributeId):
|
||||
gain = self.getEntropy(sampleIds)
|
||||
attributeVals = []
|
||||
attributeValsCount = []
|
||||
attributeValsIds = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in attributeVals:
|
||||
attributeVals.append(val)
|
||||
attributeValsCount.append(0)
|
||||
attributeValsIds.append([])
|
||||
vid = attributeVals.index(val)
|
||||
attributeValsCount[vid] += 1
|
||||
attributeValsIds[vid].append(sid)
|
||||
# print("-gig", self.attributes[attributeId])
|
||||
for vc, vids in zip(attributeValsCount, attributeValsIds):
|
||||
# print("-gig", vids)
|
||||
gain -= vc/len(sampleIds) * self.getEntropy(vids)
|
||||
return gain
|
||||
|
||||
<p>
|
||||
def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
|
||||
attributesEntropy = [0] * len(attributeIds)
|
||||
for i, attId in zip(range(len(attributeIds)), attributeIds):
|
||||
attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
|
||||
maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
|
||||
return self.attributes[maxId], maxId
|
||||
|
||||
<p>
|
||||
def isSingleLabeled(self, sampleIds):
|
||||
label = self.labels[sampleIds[0]]
|
||||
for sid in sampleIds:
|
||||
if self.labels[sid] != label:
|
||||
return False
|
||||
return True
|
||||
|
||||
<p>
|
||||
def getLabel(self, sampleId):
|
||||
return self.labels[sampleId]
|
||||
|
||||
<p>
|
||||
def id3(self):
|
||||
sampleIds = [x for x in range(len(self.sample))]
|
||||
attributeIds = [x for x in range(len(self.attributes))]
|
||||
self.root = self.id3Recv(sampleIds, attributeIds, self.root)
|
||||
|
||||
<p>
|
||||
def id3Recv(self, sampleIds, attributeIds, root):
|
||||
root = Node() # Initialize current root
|
||||
if self.isSingleLabeled(sampleIds):
|
||||
root.value = self.labels[sampleIds[0]]
|
||||
return root
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) == 0:
|
||||
root.value = self.getDominantLabel(sampleIds)
|
||||
return root
|
||||
bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
|
||||
sampleIds, attributeIds)
|
||||
# print(bestAttrName)
|
||||
root.value = bestAttrName
|
||||
root.childs = [] # Create list of children
|
||||
for value in self.getAttributeValues(sampleIds, bestAttrId):
|
||||
# print(value)
|
||||
child = Node()
|
||||
child.value = value
|
||||
root.childs.append(child) # Append new child node to current
|
||||
# root
|
||||
childSampleIds = []
|
||||
for sid in sampleIds:
|
||||
if self.sample[sid][bestAttrId] == value:
|
||||
childSampleIds.append(sid)
|
||||
if len(childSampleIds) == 0:
|
||||
child.next = self.getDominantLabel(sampleIds)
|
||||
else:
|
||||
# print(bestAttrName, bestAttrId)
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) > 0 and bestAttrId in attributeIds:
|
||||
toRemove = attributeIds.index(bestAttrId)
|
||||
attributeIds.pop(toRemove)
|
||||
child.next = self.id3Recv(
|
||||
childSampleIds, attributeIds, child.next)
|
||||
return root
|
||||
|
||||
<p>
|
||||
def printTree(self):
|
||||
if self.root:
|
||||
roots = deque()
|
||||
roots.append(self.root)
|
||||
while len(roots) > 0:
|
||||
root = roots.popleft()
|
||||
print(root.value)
|
||||
if root.childs:
|
||||
for child in root.childs:
|
||||
print('({})'.format(child.value))
|
||||
roots.append(child.next)
|
||||
elif root.next:
|
||||
print(root.next)
|
||||
|
||||
<p>
|
||||
def test():
|
||||
f = open('DataFiles/rideclass.csv')
|
||||
attributes = f.readline().split(',')
|
||||
attributes = attributes[1:len(attributes)-1]
|
||||
print(attributes)
|
||||
sample = f.readlines()
|
||||
f.close()
|
||||
for i in range(len(sample)):
|
||||
sample[i] = re.sub('\d+,', '', sample[i])
|
||||
sample[i] = sample[i].strip().split(',')
|
||||
labels = []
|
||||
for s in sample:
|
||||
labels.append(s.pop())
|
||||
# print(sample)
|
||||
# print(labels)
|
||||
decisionTree = DecisionTree(sample, attributes, labels)
|
||||
print("System entropy {}".format(decisionTree.entropy))
|
||||
decisionTree.id3()
|
||||
decisionTree.printTree()
|
||||
|
||||
<p>
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec18">Cancer Data again now with Decision Trees </h2>
|
||||
<h2 id="___sec22">Cancer Data again now with Decision Trees and other Methods </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
@@ -887,7 +1160,7 @@ deep_tree_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec19">Another example, the moons again </h2>
|
||||
<h2 id="___sec23">Another example, the moons again </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
@@ -959,7 +1232,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec20">Playing around with regions </h2>
|
||||
<h2 id="___sec24">Playing around with regions </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
@@ -967,7 +1240,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
Xs <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>, <span style="color: #666666">2</span>) <span style="color: #666666">-</span> <span style="color: #666666">0.5</span>
|
||||
ys <span style="color: #666666">=</span> (Xs[:, <span style="color: #666666">0</span>] <span style="color: #666666">></span> <span style="color: #666666">0</span>)<span style="color: #666666">.</span>astype(np<span style="color: #666666">.</span>float32) <span style="color: #666666">*</span> <span style="color: #666666">2</span>
|
||||
|
||||
angle <span style="color: #666666">=</span> np<span style="color: #666666">.</span>pi <span style="color: #666666">/</span> <span style="color: #666666">4</span>
|
||||
angle <span style="color: #666666">=</span> np<span style="color: #666666">.</span>pi<span style="color: #666666">/4</span>
|
||||
rotation_matrix <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[np<span style="color: #666666">.</span>cos(angle), <span style="color: #666666">-</span>np<span style="color: #666666">.</span>sin(angle)], [np<span style="color: #666666">.</span>sin(angle), np<span style="color: #666666">.</span>cos(angle)]])
|
||||
Xsr <span style="color: #666666">=</span> Xs<span style="color: #666666">.</span>dot(rotation_matrix)
|
||||
|
||||
@@ -987,7 +1260,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec21">Regression trees </h2>
|
||||
<h2 id="___sec25">Regression trees </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
@@ -1009,7 +1282,7 @@ tree_reg<span style="color: #666666">.</span>fit(X, y)
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec22">Final regressor code </h2>
|
||||
<h2 id="___sec26">Final regressor code </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
@@ -1087,7 +1360,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec23">Pros and cons of trees, pros </h2>
|
||||
<h2 id="___sec27">Pros and cons of trees, pros </h2>
|
||||
|
||||
<ul>
|
||||
<li> White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines)</li>
|
||||
@@ -1101,7 +1374,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec24">Disadvantages </h2>
|
||||
<h2 id="___sec28">Disadvantages </h2>
|
||||
|
||||
<ul>
|
||||
<li> Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches</li>
|
||||
@@ -1118,7 +1391,7 @@ However, by aggregating many decision trees, using methods like bagging, random
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec25">Bagging </h2>
|
||||
<h2 id="___sec29">Bagging </h2>
|
||||
|
||||
<p>
|
||||
The <b>plain</b> decision trees suffer from high
|
||||
@@ -1134,6 +1407,11 @@ of \( n \) to \( p \) is moderately large.
|
||||
general-purpose procedure for reducing the variance of a statistical
|
||||
learning method.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec30">More bagging </h2>
|
||||
|
||||
<p>
|
||||
Bagging typically results in improved accuracy
|
||||
over prediction using a single tree. Unfortunately, however, it can be
|
||||
@@ -1161,7 +1439,7 @@ predictor, averaged over all \( B \) trees.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec26">Simple example, head or tail </h2>
|
||||
<h2 id="___sec31">Simple example, head or tail </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
@@ -1181,7 +1459,58 @@ plt<span style="color: #666666">.</span>show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec27">Random forests </h2>
|
||||
<h2 id="___sec32">Bagging Example </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_moons
|
||||
|
||||
X, y <span style="color: #666666">=</span> make_moons(n_samples<span style="color: #666666">=500</span>, noise<span style="color: #666666">=0.30</span>, random_state<span style="color: #666666">=42</span>)
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(X, y, random_state<span style="color: #666666">=42</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> VotingClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
||||
|
||||
log_clf <span style="color: #666666">=</span> LogisticRegression(solver<span style="color: #666666">=</span><span style="color: #BA2121">"liblinear"</span>, random_state<span style="color: #666666">=42</span>)
|
||||
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=10</span>, random_state<span style="color: #666666">=42</span>)
|
||||
svm_clf <span style="color: #666666">=</span> SVC(gamma<span style="color: #666666">=</span><span style="color: #BA2121">"auto"</span>, random_state<span style="color: #666666">=42</span>)
|
||||
|
||||
voting_clf <span style="color: #666666">=</span> VotingClassifier(
|
||||
estimators<span style="color: #666666">=</span>[(<span style="color: #BA2121">'lr'</span>, log_clf), (<span style="color: #BA2121">'rf'</span>, rnd_clf), (<span style="color: #BA2121">'svc'</span>, svm_clf)],
|
||||
voting<span style="color: #666666">=</span><span style="color: #BA2121">'hard'</span>)
|
||||
|
||||
voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> clf <span style="color: #AA22FF; font-weight: bold">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(clf<span style="color: #666666">.</span><span style="color: #19177C">__class__</span><span style="color: #666666">.</span><span style="color: #19177C">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
|
||||
log_clf <span style="color: #666666">=</span> LogisticRegression(solver<span style="color: #666666">=</span><span style="color: #BA2121">"liblinear"</span>, random_state<span style="color: #666666">=42</span>)
|
||||
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=10</span>, random_state<span style="color: #666666">=42</span>)
|
||||
svm_clf <span style="color: #666666">=</span> SVC(gamma<span style="color: #666666">=</span><span style="color: #BA2121">"auto"</span>, probability<span style="color: #666666">=</span><span style="color: #008000">True</span>, random_state<span style="color: #666666">=42</span>)
|
||||
|
||||
voting_clf <span style="color: #666666">=</span> VotingClassifier(
|
||||
estimators<span style="color: #666666">=</span>[(<span style="color: #BA2121">'lr'</span>, log_clf), (<span style="color: #BA2121">'rf'</span>, rnd_clf), (<span style="color: #BA2121">'svc'</span>, svm_clf)],
|
||||
voting<span style="color: #666666">=</span><span style="color: #BA2121">'soft'</span>)
|
||||
voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> clf <span style="color: #AA22FF; font-weight: bold">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(clf<span style="color: #666666">.</span><span style="color: #19177C">__class__</span><span style="color: #666666">.</span><span style="color: #19177C">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec33">Random forests </h2>
|
||||
|
||||
<p>
|
||||
Random forests provide an improvement over bagged trees by way of a
|
||||
@@ -1225,7 +1554,7 @@ this setting.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec28">A simple scikit-learn example </h2>
|
||||
<h2 id="___sec34">A simple scikit-learn example </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
@@ -1243,7 +1572,7 @@ accuracy <span style="color: #666666">=</span> cross_validate(Random_Forest_mode
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec29">Please, not the moons again! </h2>
|
||||
<h2 id="___sec35">Please, not the moons again! </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
@@ -1301,7 +1630,7 @@ voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec30">Bagging examples </h2>
|
||||
<h2 id="___sec36">Bagging examples </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -1362,7 +1691,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec31">Then random forests </h2>
|
||||
<h2 id="___sec37">Then random forests </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
@@ -1381,12 +1710,6 @@ rnd_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred_rf <span style="color: #666666">=</span> rnd_clf<span style="color: #666666">.</span>predict(X_test)
|
||||
np<span style="color: #666666">.</span>sum(y_pred <span style="color: #666666">==</span> y_pred_rf) <span style="color: #666666">/</span> <span style="color: #008000">len</span>(y_pred)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec32">Boosting and more </h2>
|
||||
More material to come here.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"<!-- Author: --> \n",
|
||||
"**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n",
|
||||
"\n",
|
||||
"Date: **Oct 29, 2019**\n",
|
||||
"Date: **Oct 31, 2019**\n",
|
||||
"\n",
|
||||
"Copyright 1999-2019, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license\n",
|
||||
"\n",
|
||||
@@ -506,9 +506,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## The CART (Classification and Regression Tree) algorithm\n",
|
||||
"\n",
|
||||
"The above functions (gini, entropy and misclassification error) are important components of the so-called CART algorithm. We will discuss this algorithm first before we move on to the information gain algorithm ID3."
|
||||
"## Visualizing the Tree, Classification"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -519,62 +517,210 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from random import seed\n",
|
||||
"from random import randrange\n",
|
||||
"from csv import reader\n",
|
||||
" \n",
|
||||
"# Load a CSV file\n",
|
||||
"def load_csv(filename):\n",
|
||||
"\tfile = open(filename, \"rb\")\n",
|
||||
"\tlines = reader(file)\n",
|
||||
"\tdataset = list(lines)\n",
|
||||
"\treturn dataset\n",
|
||||
" \n",
|
||||
"# Convert string column to float\n",
|
||||
"def str_column_to_float(dataset, column):\n",
|
||||
"\tfor row in dataset:\n",
|
||||
"\t\trow[column] = float(row[column].strip())\n",
|
||||
" \n",
|
||||
"# Split a dataset into k folds\n",
|
||||
"def cross_validation_split(dataset, n_folds):\n",
|
||||
"\tdataset_split = list()\n",
|
||||
"\tdataset_copy = list(dataset)\n",
|
||||
"\tfold_size = int(len(dataset) / n_folds)\n",
|
||||
"\tfor i in range(n_folds):\n",
|
||||
"\t\tfold = list()\n",
|
||||
"\t\twhile len(fold) < fold_size:\n",
|
||||
"\t\t\tindex = randrange(len(dataset_copy))\n",
|
||||
"\t\t\tfold.append(dataset_copy.pop(index))\n",
|
||||
"\t\tdataset_split.append(fold)\n",
|
||||
"\treturn dataset_split\n",
|
||||
" \n",
|
||||
"# Calculate accuracy percentage\n",
|
||||
"def accuracy_metric(actual, predicted):\n",
|
||||
"\tcorrect = 0\n",
|
||||
"\tfor i in range(len(actual)):\n",
|
||||
"\t\tif actual[i] == predicted[i]:\n",
|
||||
"\t\t\tcorrect += 1\n",
|
||||
"\treturn correct / float(len(actual)) * 100.0\n",
|
||||
" \n",
|
||||
"# Evaluate an algorithm using a cross validation split\n",
|
||||
"def evaluate_algorithm(dataset, algorithm, n_folds, *args):\n",
|
||||
"\tfolds = cross_validation_split(dataset, n_folds)\n",
|
||||
"\tscores = list()\n",
|
||||
"\tfor fold in folds:\n",
|
||||
"\t\ttrain_set = list(folds)\n",
|
||||
"\t\ttrain_set.remove(fold)\n",
|
||||
"\t\ttrain_set = sum(train_set, [])\n",
|
||||
"\t\ttest_set = list()\n",
|
||||
"\t\tfor row in fold:\n",
|
||||
"\t\t\trow_copy = list(row)\n",
|
||||
"\t\t\ttest_set.append(row_copy)\n",
|
||||
"\t\t\trow_copy[-1] = None\n",
|
||||
"\t\tpredicted = algorithm(train_set, test_set, *args)\n",
|
||||
"\t\tactual = [row[-1] for row in fold]\n",
|
||||
"\t\taccuracy = accuracy_metric(actual, predicted)\n",
|
||||
"\t\tscores.append(accuracy)\n",
|
||||
"\treturn scores\n",
|
||||
" \n",
|
||||
"import os\n",
|
||||
"from sklearn.datasets import load_breast_cancer\n",
|
||||
"from sklearn.tree import DecisionTreeClassifier\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.metrics import confusion_matrix\n",
|
||||
"from sklearn.tree import export_graphviz\n",
|
||||
"\n",
|
||||
"from IPython.display import Image \n",
|
||||
"from pydot import graph_from_dot_data\n",
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"cancer = load_breast_cancer()\n",
|
||||
"X = pd.DataFrame(cancer.data, columns=cancer.feature_names)\n",
|
||||
"print(X)\n",
|
||||
"y = pd.Categorical.from_codes(cancer.target, cancer.target_names)\n",
|
||||
"y = pd.get_dummies(y)\n",
|
||||
"print(y)\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)\n",
|
||||
"tree_clf = DecisionTreeClassifier(max_depth=5)\n",
|
||||
"tree_clf.fit(X_train, y_train)\n",
|
||||
"\n",
|
||||
"export_graphviz(\n",
|
||||
" tree_clf,\n",
|
||||
" out_file=\"DataFiles/cancer.dot\",\n",
|
||||
" feature_names=cancer.feature_names,\n",
|
||||
" class_names=cancer.target_names,\n",
|
||||
" rounded=True,\n",
|
||||
" filled=True\n",
|
||||
")\n",
|
||||
"cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'\n",
|
||||
"os.system(cmd)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Visualizing the Tree, The Moons"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Common imports\n",
|
||||
"import numpy as np\n",
|
||||
"from sklearn.model_selection import train_test_split \n",
|
||||
"from sklearn.tree import DecisionTreeClassifier\n",
|
||||
"from sklearn.datasets import make_moons\n",
|
||||
"from sklearn.tree import export_graphviz\n",
|
||||
"from pydot import graph_from_dot_data\n",
|
||||
"import pandas as pd\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"np.random.seed(42)\n",
|
||||
"X, y = make_moons(n_samples=100, noise=0.25, random_state=53)\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)\n",
|
||||
"tree_clf = DecisionTreeClassifier(max_depth=5)\n",
|
||||
"tree_clf.fit(X_train, y_train)\n",
|
||||
"\n",
|
||||
"export_graphviz(\n",
|
||||
" tree_clf,\n",
|
||||
" out_file=\"DataFiles/moons.dot\",\n",
|
||||
" rounded=True,\n",
|
||||
" filled=True\n",
|
||||
")\n",
|
||||
"cmd = 'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png'\n",
|
||||
"os.system(cmd)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Computing the Gini index\n",
|
||||
"\n",
|
||||
"The example we will look at is a classical one in many Machine\n",
|
||||
"Learning applications. Based on various meteorological features, we\n",
|
||||
"have several so-called attributes which decide whether we at the end\n",
|
||||
"will do some outdoor activity like skiing, going for a bike ride etc\n",
|
||||
"etc. The table here contains the feautures **outlook**, **temperature**,\n",
|
||||
"**humidity** and **wind**. The target or output is whether we ride\n",
|
||||
"(True=1) or whether we do something else that day (False=0). The\n",
|
||||
"attributes for each feature are then sunny, overcast and rain for the\n",
|
||||
"outlook, hot, cold and mild for temperature, high and normal for\n",
|
||||
"humidity and weak and strong for wind.\n",
|
||||
"\n",
|
||||
"The table here summarizes the various attributes and\n",
|
||||
"<table border=\"1\">\n",
|
||||
"<thead>\n",
|
||||
"<tr><th align=\"center\">Day</th> <th align=\"center\">Outlook </th> <th align=\"center\">Temperature</th> <th align=\"center\">Humidity</th> <th align=\"center\"> Wind </th> <th align=\"center\">Ride</th> </tr>\n",
|
||||
"</thead>\n",
|
||||
"<tbody>\n",
|
||||
"<tr><td align=\"center\"> 1 </td> <td align=\"center\"> Sunny </td> <td align=\"center\"> Hot </td> <td align=\"center\"> High </td> <td align=\"center\"> Weak </td> <td align=\"center\"> 0 </td> </tr>\n",
|
||||
"<tr><td align=\"center\"> 2 </td> <td align=\"center\"> Sunny </td> <td align=\"center\"> Hot </td> <td align=\"center\"> High </td> <td align=\"center\"> Strong </td> <td align=\"center\"> 1 </td> </tr>\n",
|
||||
"<tr><td align=\"center\"> 3 </td> <td align=\"center\"> Overcast </td> <td align=\"center\"> Hot </td> <td align=\"center\"> High </td> <td align=\"center\"> Weak </td> <td align=\"center\"> 1 </td> </tr>\n",
|
||||
"<tr><td align=\"center\"> 4 </td> <td align=\"center\"> Rain </td> <td align=\"center\"> Mild </td> <td align=\"center\"> High </td> <td align=\"center\"> Weak </td> <td align=\"center\"> 1 </td> </tr>\n",
|
||||
"<tr><td align=\"center\"> 5 </td> <td align=\"center\"> Rain </td> <td align=\"center\"> Cool </td> <td align=\"center\"> Normal </td> <td align=\"center\"> Weak </td> <td align=\"center\"> 1 </td> </tr>\n",
|
||||
"<tr><td align=\"center\"> 6 </td> <td align=\"center\"> Rain </td> <td align=\"center\"> Cool </td> <td align=\"center\"> Normal </td> <td align=\"center\"> Strong </td> <td align=\"center\"> 0 </td> </tr>\n",
|
||||
"<tr><td align=\"center\"> 7 </td> <td align=\"center\"> Overcast </td> <td align=\"center\"> Cool </td> <td align=\"center\"> Normal </td> <td align=\"center\"> Strong </td> <td align=\"center\"> 1 </td> </tr>\n",
|
||||
"<tr><td align=\"center\"> 8 </td> <td align=\"center\"> Sunny </td> <td align=\"center\"> Mild </td> <td align=\"center\"> High </td> <td align=\"center\"> Weak </td> <td align=\"center\"> 0 </td> </tr>\n",
|
||||
"<tr><td align=\"center\"> 9 </td> <td align=\"center\"> Sunny </td> <td align=\"center\"> Cool </td> <td align=\"center\"> Normal </td> <td align=\"center\"> Weak </td> <td align=\"center\"> 1 </td> </tr>\n",
|
||||
"<tr><td align=\"center\"> 10 </td> <td align=\"center\"> Rain </td> <td align=\"center\"> Mild </td> <td align=\"center\"> Normal </td> <td align=\"center\"> Weak </td> <td align=\"center\"> 1 </td> </tr>\n",
|
||||
"<tr><td align=\"center\"> 11 </td> <td align=\"center\"> Sunny </td> <td align=\"center\"> Mild </td> <td align=\"center\"> Normal </td> <td align=\"center\"> Strong </td> <td align=\"center\"> 1 </td> </tr>\n",
|
||||
"<tr><td align=\"center\"> 12 </td> <td align=\"center\"> Overcast </td> <td align=\"center\"> Mild </td> <td align=\"center\"> High </td> <td align=\"center\"> Strong </td> <td align=\"center\"> 1 </td> </tr>\n",
|
||||
"<tr><td align=\"center\"> 13 </td> <td align=\"center\"> Overcast </td> <td align=\"center\"> Hot </td> <td align=\"center\"> Normal </td> <td align=\"center\"> Weak </td> <td align=\"center\"> 1 </td> </tr>\n",
|
||||
"<tr><td align=\"center\"> 14 </td> <td align=\"center\"> Rain </td> <td align=\"center\"> Mild </td> <td align=\"center\"> High </td> <td align=\"center\"> Strong </td> <td align=\"center\"> 0 </td> </tr>\n",
|
||||
"</tbody>\n",
|
||||
"</table>\n",
|
||||
"\n",
|
||||
"## Simple Python Code to read in Data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Common imports\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
|
||||
"from sklearn.compose import ColumnTransformer\n",
|
||||
"from IPython.display import Image \n",
|
||||
"from pydot import graph_from_dot_data\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Where to save the figures and data files\n",
|
||||
"PROJECT_ROOT_DIR = \"Results\"\n",
|
||||
"FIGURE_ID = \"Results/FigureFiles\"\n",
|
||||
"DATA_ID = \"DataFiles/\"\n",
|
||||
"\n",
|
||||
"if not os.path.exists(PROJECT_ROOT_DIR):\n",
|
||||
" os.mkdir(PROJECT_ROOT_DIR)\n",
|
||||
"\n",
|
||||
"if not os.path.exists(FIGURE_ID):\n",
|
||||
" os.makedirs(FIGURE_ID)\n",
|
||||
"\n",
|
||||
"if not os.path.exists(DATA_ID):\n",
|
||||
" os.makedirs(DATA_ID)\n",
|
||||
"\n",
|
||||
"def image_path(fig_id):\n",
|
||||
" return os.path.join(FIGURE_ID, fig_id)\n",
|
||||
"\n",
|
||||
"def data_path(dat_id):\n",
|
||||
" return os.path.join(DATA_ID, dat_id)\n",
|
||||
"\n",
|
||||
"def save_fig(fig_id):\n",
|
||||
" plt.savefig(image_path(fig_id) + \".png\", format='png')\n",
|
||||
"\n",
|
||||
"infile = open(data_path(\"ride.csv\"),'r')\n",
|
||||
"\n",
|
||||
"# Read the experimental data with Pandas\n",
|
||||
"from IPython.display import display\n",
|
||||
"ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))\n",
|
||||
"ridedata = pd.DataFrame(ridedata)\n",
|
||||
"display(ridedata)\n",
|
||||
"# Features and targets\n",
|
||||
"X = ridedata.loc[:, ridedata.columns != 'Ride'].values\n",
|
||||
"display(X)\n",
|
||||
"y = ridedata.loc[:, ridedata.columns == 'Ride'].values\n",
|
||||
"display(y)\n",
|
||||
"# Categorical variables to one-hot's\n",
|
||||
"onehotencoder = OneHotEncoder(categories=\"auto\")\n",
|
||||
"\n",
|
||||
"X = ColumnTransformer([(\"\", onehotencoder)]).fit_transform(X)\n",
|
||||
"y.shape\n",
|
||||
"\n",
|
||||
"display(X)\n",
|
||||
"display(y)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Computing the Gini Factor\n",
|
||||
"\n",
|
||||
"The above functions (gini, entropy and misclassification error) are\n",
|
||||
"important components of the so-called CART algorithm. We will discuss\n",
|
||||
"this algorithm below after we have discussed the information gain\n",
|
||||
"algorithm ID3.\n",
|
||||
"\n",
|
||||
"In the example here we have converted all our attributes into numerical values $0,1,2$ etc."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Split a dataset based on an attribute and an attribute value\n",
|
||||
"def test_split(index, value, dataset):\n",
|
||||
"\tleft, right = list(), list()\n",
|
||||
@@ -604,7 +750,7 @@
|
||||
"\t\t# weight the group score by its relative size\n",
|
||||
"\t\tgini += (1.0 - score) * (size / n_instances)\n",
|
||||
"\treturn gini\n",
|
||||
" \n",
|
||||
"\n",
|
||||
"# Select the best split point for a dataset\n",
|
||||
"def get_split(dataset):\n",
|
||||
"\tclass_values = list(set(row[-1] for row in dataset))\n",
|
||||
@@ -613,83 +759,28 @@
|
||||
"\t\tfor row in dataset:\n",
|
||||
"\t\t\tgroups = test_split(index, row[index], dataset)\n",
|
||||
"\t\t\tgini = gini_index(groups, class_values)\n",
|
||||
"\t\t\tprint('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini))\n",
|
||||
"\t\t\tif gini < b_score:\n",
|
||||
"\t\t\t\tb_index, b_value, b_score, b_groups = index, row[index], gini, groups\n",
|
||||
"\treturn {'index':b_index, 'value':b_value, 'groups':b_groups}\n",
|
||||
" \n",
|
||||
"# Create a terminal node value\n",
|
||||
"def to_terminal(group):\n",
|
||||
"\toutcomes = [row[-1] for row in group]\n",
|
||||
"\treturn max(set(outcomes), key=outcomes.count)\n",
|
||||
" \n",
|
||||
"# Create child splits for a node or make terminal\n",
|
||||
"def split(node, max_depth, min_size, depth):\n",
|
||||
"\tleft, right = node['groups']\n",
|
||||
"\tdel(node['groups'])\n",
|
||||
"\t# check for a no split\n",
|
||||
"\tif not left or not right:\n",
|
||||
"\t\tnode['left'] = node['right'] = to_terminal(left + right)\n",
|
||||
"\t\treturn\n",
|
||||
"\t# check for max depth\n",
|
||||
"\tif depth >= max_depth:\n",
|
||||
"\t\tnode['left'], node['right'] = to_terminal(left), to_terminal(right)\n",
|
||||
"\t\treturn\n",
|
||||
"\t# process left child\n",
|
||||
"\tif len(left) <= min_size:\n",
|
||||
"\t\tnode['left'] = to_terminal(left)\n",
|
||||
"\telse:\n",
|
||||
"\t\tnode['left'] = get_split(left)\n",
|
||||
"\t\tsplit(node['left'], max_depth, min_size, depth+1)\n",
|
||||
"\t# process right child\n",
|
||||
"\tif len(right) <= min_size:\n",
|
||||
"\t\tnode['right'] = to_terminal(right)\n",
|
||||
"\telse:\n",
|
||||
"\t\tnode['right'] = get_split(right)\n",
|
||||
"\t\tsplit(node['right'], max_depth, min_size, depth+1)\n",
|
||||
" \n",
|
||||
"# Build a decision tree\n",
|
||||
"def build_tree(train, max_depth, min_size):\n",
|
||||
"\troot = get_split(train)\n",
|
||||
"\tsplit(root, max_depth, min_size, 1)\n",
|
||||
"\treturn root\n",
|
||||
" \n",
|
||||
"# Make a prediction with a decision tree\n",
|
||||
"def predict(node, row):\n",
|
||||
"\tif row[node['index']] < node['value']:\n",
|
||||
"\t\tif isinstance(node['left'], dict):\n",
|
||||
"\t\t\treturn predict(node['left'], row)\n",
|
||||
"\t\telse:\n",
|
||||
"\t\t\treturn node['left']\n",
|
||||
"\telse:\n",
|
||||
"\t\tif isinstance(node['right'], dict):\n",
|
||||
"\t\t\treturn predict(node['right'], row)\n",
|
||||
"\t\telse:\n",
|
||||
"\t\t\treturn node['right']\n",
|
||||
" \n",
|
||||
"# Classification and Regression Tree Algorithm\n",
|
||||
"def decision_tree(train, test, max_depth, min_size):\n",
|
||||
"\ttree = build_tree(train, max_depth, min_size)\n",
|
||||
"\tpredictions = list()\n",
|
||||
"\tfor row in test:\n",
|
||||
"\t\tprediction = predict(tree, row)\n",
|
||||
"\t\tpredictions.append(prediction)\n",
|
||||
"\treturn(predictions)\n",
|
||||
" \n",
|
||||
"# Test CART \n",
|
||||
"seed(1)\n",
|
||||
"# load and prepare data\n",
|
||||
"filename = 'DataFiles/rideclass.csv'\n",
|
||||
"dataset = load_csv(filename)\n",
|
||||
"# convert string attributes to integers\n",
|
||||
"for i in range(len(dataset[0])):\n",
|
||||
"\tstr_column_to_float(dataset, i)\n",
|
||||
"# evaluate algorithm\n",
|
||||
"n_folds = 5\n",
|
||||
"max_depth = 5\n",
|
||||
"min_size = 10\n",
|
||||
"scores = evaluate_algorithm(dataset, decision_tree, n_folds, max_depth, min_size)\n",
|
||||
"print('Scores: %s' % scores)\n",
|
||||
"print('Mean Accuracy: %.3f%%' % (sum(scores)/float(len(scores))))"
|
||||
"dataset = [[0,0,0,0,0],\n",
|
||||
" [0,0,0,1,1],\n",
|
||||
" [1,0,0,0,1],\n",
|
||||
" [2,1,0,0,1],\n",
|
||||
" [2,2,1,0,1],\n",
|
||||
" [2,2,1,1,0],\n",
|
||||
" [1,2,1,1,1],\n",
|
||||
" [0,1,0,0,0],\n",
|
||||
" [0,2,1,0,1],\n",
|
||||
" [2,1,1,0,1],\n",
|
||||
" [0,1,1,1,1],\n",
|
||||
" [1,1,0,1,1],\n",
|
||||
" [1,0,1,0,1],\n",
|
||||
" [2,1,0,1,0]]\n",
|
||||
"\n",
|
||||
"split = get_split(dataset)\n",
|
||||
"print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -729,14 +820,201 @@
|
||||
"\n",
|
||||
"## Implementing the ID3 Algorithm\n",
|
||||
"\n",
|
||||
"**more text to come here**, material presented during lecture Friday Oct 25.\n",
|
||||
"import re\n",
|
||||
"import math\n",
|
||||
"from collections import deque\n",
|
||||
"\n",
|
||||
"## Cancer Data again now with Decision Trees"
|
||||
"<!-- x is examples in training set -->\n",
|
||||
"<!-- y is set of attributes -->\n",
|
||||
"<!-- label is target attributes -->\n",
|
||||
"<!-- Node is a class which has properties values, childs, and next -->\n",
|
||||
"<!-- root is top node in the decision tree -->\n",
|
||||
"\n",
|
||||
"class Node(object):\n",
|
||||
"\tdef __init__(self):\n",
|
||||
"\t\tself.value = None\n",
|
||||
"\t\tself.next = None\n",
|
||||
"\t\tself.childs = None\n",
|
||||
"\n",
|
||||
"<!-- Simple class of Decision Tree -->\n",
|
||||
"<!-- Aimed for who want to learn Decision Tree, so it is not optimized -->\n",
|
||||
"class DecisionTree(object):\n",
|
||||
"\tdef __init__(self, sample, attributes, labels):\n",
|
||||
"\t\tself.sample = sample\n",
|
||||
"\t\tself.attributes = attributes\n",
|
||||
"\t\tself.labels = labels\n",
|
||||
"\t\tself.labelCodes = None\n",
|
||||
"\t\tself.labelCodesCount = None\n",
|
||||
"\t\tself.initLabelCodes()\n",
|
||||
"\t\t# print(self.labelCodes)\n",
|
||||
"\t\tself.root = None\n",
|
||||
"\t\tself.entropy = self.getEntropy([x for x in range(len(self.labels))])\n",
|
||||
"\n",
|
||||
"\tdef initLabelCodes(self):\n",
|
||||
"\t\tself.labelCodes = []\n",
|
||||
"\t\tself.labelCodesCount = []\n",
|
||||
"\t\tfor l in self.labels:\n",
|
||||
"\t\t\tif l not in self.labelCodes:\n",
|
||||
"\t\t\t\tself.labelCodes.append(l)\n",
|
||||
"\t\t\t\tself.labelCodesCount.append(0)\n",
|
||||
"\t\t\tself.labelCodesCount[self.labelCodes.index(l)] += 1\n",
|
||||
"\n",
|
||||
"\tdef getLabelCodeId(self, sampleId):\n",
|
||||
"\t\treturn self.labelCodes.index(self.labels[sampleId])\n",
|
||||
"\n",
|
||||
"\tdef getAttributeValues(self, sampleIds, attributeId):\n",
|
||||
"\t\tvals = []\n",
|
||||
"\t\tfor sid in sampleIds:\n",
|
||||
"\t\t\tval = self.sample[sid][attributeId]\n",
|
||||
"\t\t\tif val not in vals:\n",
|
||||
"\t\t\t\tvals.append(val)\n",
|
||||
"\t\t# print(vals)\n",
|
||||
"\t\treturn vals\n",
|
||||
"\n",
|
||||
"\tdef getEntropy(self, sampleIds):\n",
|
||||
"\t\tentropy = 0\n",
|
||||
"\t\tlabelCount = [0] * len(self.labelCodes)\n",
|
||||
"\t\tfor sid in sampleIds:\n",
|
||||
"\t\t\tlabelCount[self.getLabelCodeId(sid)] += 1\n",
|
||||
"\t\t# print(\"-ge\", labelCount)\n",
|
||||
"\t\tfor lv in labelCount:\n",
|
||||
"\t\t\t# print(lv)\n",
|
||||
"\t\t\tif lv != 0:\n",
|
||||
"\t\t\t\tentropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)\n",
|
||||
"\t\t\telse:\n",
|
||||
"\t\t\t\tentropy += 0\n",
|
||||
"\t\treturn entropy\n",
|
||||
"\n",
|
||||
"\tdef getDominantLabel(self, sampleIds):\n",
|
||||
"\t\tlabelCodesCount = [0] * len(self.labelCodes)\n",
|
||||
"\t\tfor sid in sampleIds:\n",
|
||||
"\t\t\tlabelCodesCount[self.labelCodes.index(self.labels[sid])] += 1\n",
|
||||
"\t\treturn self.labelCodes[labelCodesCount.index(max(labelCodesCount))]\n",
|
||||
"\n",
|
||||
"\tdef getInformationGain(self, sampleIds, attributeId):\n",
|
||||
"\t\tgain = self.getEntropy(sampleIds)\n",
|
||||
"\t\tattributeVals = []\n",
|
||||
"\t\tattributeValsCount = []\n",
|
||||
"\t\tattributeValsIds = []\n",
|
||||
"\t\tfor sid in sampleIds:\n",
|
||||
"\t\t\tval = self.sample[sid][attributeId]\n",
|
||||
"\t\t\tif val not in attributeVals:\n",
|
||||
"\t\t\t\tattributeVals.append(val)\n",
|
||||
"\t\t\t\tattributeValsCount.append(0)\n",
|
||||
"\t\t\t\tattributeValsIds.append([])\n",
|
||||
"\t\t\tvid = attributeVals.index(val)\n",
|
||||
"\t\t\tattributeValsCount[vid] += 1\n",
|
||||
"\t\t\tattributeValsIds[vid].append(sid)\n",
|
||||
"\t\t# print(\"-gig\", self.attributes[attributeId])\n",
|
||||
"\t\tfor vc, vids in zip(attributeValsCount, attributeValsIds):\n",
|
||||
"\t\t\t# print(\"-gig\", vids)\n",
|
||||
"\t\t\tgain -= vc/len(sampleIds) * self.getEntropy(vids)\n",
|
||||
"\t\treturn gain\n",
|
||||
"\n",
|
||||
"\tdef getAttributeMaxInformationGain(self, sampleIds, attributeIds):\n",
|
||||
"\t\tattributesEntropy = [0] * len(attributeIds)\n",
|
||||
"\t\tfor i, attId in zip(range(len(attributeIds)), attributeIds):\n",
|
||||
"\t\t\tattributesEntropy[i] = self.getInformationGain(sampleIds, attId)\n",
|
||||
"\t\tmaxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]\n",
|
||||
"\t\treturn self.attributes[maxId], maxId\n",
|
||||
"\n",
|
||||
"\tdef isSingleLabeled(self, sampleIds):\n",
|
||||
"\t\tlabel = self.labels[sampleIds[0]]\n",
|
||||
"\t\tfor sid in sampleIds:\n",
|
||||
"\t\t\tif self.labels[sid] != label:\n",
|
||||
"\t\t\t\treturn False\n",
|
||||
"\t\treturn True\n",
|
||||
"\n",
|
||||
"\tdef getLabel(self, sampleId):\n",
|
||||
"\t\treturn self.labels[sampleId]\n",
|
||||
"\n",
|
||||
"\tdef id3(self):\n",
|
||||
"\t\tsampleIds = [x for x in range(len(self.sample))]\n",
|
||||
"\t\tattributeIds = [x for x in range(len(self.attributes))]\n",
|
||||
"\t\tself.root = self.id3Recv(sampleIds, attributeIds, self.root)\n",
|
||||
"\n",
|
||||
"\tdef id3Recv(self, sampleIds, attributeIds, root):\n",
|
||||
"\t\troot = Node() # Initialize current root\n",
|
||||
"\t\tif self.isSingleLabeled(sampleIds):\n",
|
||||
"\t\t\troot.value = self.labels[sampleIds[0]]\n",
|
||||
"\t\t\treturn root\n",
|
||||
"\t\t# print(attributeIds)\n",
|
||||
"\t\tif len(attributeIds) == 0:\n",
|
||||
"\t\t\troot.value = self.getDominantLabel(sampleIds)\n",
|
||||
"\t\t\treturn root\n",
|
||||
"\t\tbestAttrName, bestAttrId = self.getAttributeMaxInformationGain(\n",
|
||||
"\t\t\tsampleIds, attributeIds)\n",
|
||||
"\t\t# print(bestAttrName)\n",
|
||||
"\t\troot.value = bestAttrName\n",
|
||||
"\t\troot.childs = [] # Create list of children\n",
|
||||
"\t\tfor value in self.getAttributeValues(sampleIds, bestAttrId):\n",
|
||||
"\t\t\t# print(value)\n",
|
||||
"\t\t\tchild = Node()\n",
|
||||
"\t\t\tchild.value = value\n",
|
||||
"\t\t\troot.childs.append(child) # Append new child node to current\n",
|
||||
"\t\t\t\t\t\t\t\t\t # root\n",
|
||||
"\t\t\tchildSampleIds = []\n",
|
||||
"\t\t\tfor sid in sampleIds:\n",
|
||||
"\t\t\t\tif self.sample[sid][bestAttrId] == value:\n",
|
||||
"\t\t\t\t\tchildSampleIds.append(sid)\n",
|
||||
"\t\t\tif len(childSampleIds) == 0:\n",
|
||||
"\t\t\t\tchild.next = self.getDominantLabel(sampleIds)\n",
|
||||
"\t\t\telse:\n",
|
||||
"\t\t\t\t# print(bestAttrName, bestAttrId)\n",
|
||||
"\t\t\t\t# print(attributeIds)\n",
|
||||
"\t\t\t\tif len(attributeIds) > 0 and bestAttrId in attributeIds:\n",
|
||||
"\t\t\t\t\ttoRemove = attributeIds.index(bestAttrId)\n",
|
||||
"\t\t\t\t\tattributeIds.pop(toRemove)\n",
|
||||
"\t\t\t\tchild.next = self.id3Recv(\n",
|
||||
"\t\t\t\t\tchildSampleIds, attributeIds, child.next)\n",
|
||||
"\t\treturn root\n",
|
||||
"\n",
|
||||
"\tdef printTree(self):\n",
|
||||
"\t\tif self.root:\n",
|
||||
"\t\t\troots = deque()\n",
|
||||
"\t\t\troots.append(self.root)\n",
|
||||
"\t\t\twhile len(roots) > 0:\n",
|
||||
"\t\t\t\troot = roots.popleft()\n",
|
||||
"\t\t\t\tprint(root.value)\n",
|
||||
"\t\t\t\tif root.childs:\n",
|
||||
"\t\t\t\t\tfor child in root.childs:\n",
|
||||
"\t\t\t\t\t\tprint('({})'.format(child.value))\n",
|
||||
"\t\t\t\t\t\troots.append(child.next)\n",
|
||||
"\t\t\t\telif root.next:\n",
|
||||
"\t\t\t\t\tprint(root.next)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def test():\n",
|
||||
"\tf = open('DataFiles/rideclass.csv')\n",
|
||||
"\tattributes = f.readline().split(',')\n",
|
||||
"\tattributes = attributes[1:len(attributes)-1]\n",
|
||||
"\tprint(attributes)\n",
|
||||
"\tsample = f.readlines()\n",
|
||||
"\tf.close()\n",
|
||||
"\tfor i in range(len(sample)):\n",
|
||||
"\t\tsample[i] = re.sub('\\d+,', '', sample[i])\n",
|
||||
"\t\tsample[i] = sample[i].strip().split(',')\n",
|
||||
"\tlabels = []\n",
|
||||
"\tfor s in sample:\n",
|
||||
"\t\tlabels.append(s.pop())\n",
|
||||
"\t# print(sample)\n",
|
||||
"\t# print(labels)\n",
|
||||
"\tdecisionTree = DecisionTree(sample, attributes, labels)\n",
|
||||
"\tprint(\"System entropy {}\".format(decisionTree.entropy))\n",
|
||||
"\tdecisionTree.id3()\n",
|
||||
"\tdecisionTree.printTree()\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"if __name__ == '__main__':\n",
|
||||
"\ttest()\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## Cancer Data again now with Decision Trees and other Methods"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": 6,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -794,7 +1072,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": 7,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -875,7 +1153,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": 8,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -885,7 +1163,7 @@
|
||||
"Xs = np.random.rand(100, 2) - 0.5\n",
|
||||
"ys = (Xs[:, 0] > 0).astype(np.float32) * 2\n",
|
||||
"\n",
|
||||
"angle = np.pi / 4\n",
|
||||
"angle = np.pi/4\n",
|
||||
"rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])\n",
|
||||
"Xsr = Xs.dot(rotation_matrix)\n",
|
||||
"\n",
|
||||
@@ -912,7 +1190,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": 9,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -928,7 +1206,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": 10,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -949,7 +1227,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 11,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -997,7 +1275,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": 12,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1086,6 +1364,8 @@
|
||||
"learning method. \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## More bagging\n",
|
||||
"\n",
|
||||
"Bagging typically results in improved accuracy\n",
|
||||
"over prediction using a single tree. Unfortunately, however, it can be\n",
|
||||
"difficult to interpret the resulting model. Recall that one of the\n",
|
||||
@@ -1113,7 +1393,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"execution_count": 13,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1133,6 +1413,66 @@
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Bagging Example"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.datasets import make_moons\n",
|
||||
"\n",
|
||||
"X, y = make_moons(n_samples=500, noise=0.30, random_state=42)\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)\n",
|
||||
"\n",
|
||||
"from sklearn.ensemble import RandomForestClassifier\n",
|
||||
"from sklearn.ensemble import VotingClassifier\n",
|
||||
"from sklearn.linear_model import LogisticRegression\n",
|
||||
"from sklearn.svm import SVC\n",
|
||||
"\n",
|
||||
"log_clf = LogisticRegression(solver=\"liblinear\", random_state=42)\n",
|
||||
"rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)\n",
|
||||
"svm_clf = SVC(gamma=\"auto\", random_state=42)\n",
|
||||
"\n",
|
||||
"voting_clf = VotingClassifier(\n",
|
||||
" estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n",
|
||||
" voting='hard')\n",
|
||||
"\n",
|
||||
"voting_clf.fit(X_train, y_train)\n",
|
||||
"\n",
|
||||
"from sklearn.metrics import accuracy_score\n",
|
||||
"\n",
|
||||
"for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n",
|
||||
" clf.fit(X_train, y_train)\n",
|
||||
" y_pred = clf.predict(X_test)\n",
|
||||
" print(clf.__class__.__name__, accuracy_score(y_test, y_pred))\n",
|
||||
"\n",
|
||||
"log_clf = LogisticRegression(solver=\"liblinear\", random_state=42)\n",
|
||||
"rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)\n",
|
||||
"svm_clf = SVC(gamma=\"auto\", probability=True, random_state=42)\n",
|
||||
"\n",
|
||||
"voting_clf = VotingClassifier(\n",
|
||||
" estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n",
|
||||
" voting='soft')\n",
|
||||
"voting_clf.fit(X_train, y_train)\n",
|
||||
"\n",
|
||||
"from sklearn.metrics import accuracy_score\n",
|
||||
"\n",
|
||||
"for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n",
|
||||
" clf.fit(X_train, y_train)\n",
|
||||
" y_pred = clf.predict(X_test)\n",
|
||||
" print(clf.__class__.__name__, accuracy_score(y_test, y_pred))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
@@ -1188,7 +1528,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"execution_count": 15,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1215,7 +1555,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"execution_count": 16,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1243,7 +1583,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"execution_count": 17,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1259,7 +1599,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"execution_count": 18,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1277,7 +1617,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"execution_count": 19,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1300,7 +1640,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"execution_count": 20,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1318,7 +1658,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"execution_count": 21,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1330,7 +1670,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"execution_count": 22,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1344,7 +1684,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"execution_count": 23,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1387,7 +1727,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 20,
|
||||
"execution_count": 24,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1400,7 +1740,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 21,
|
||||
"execution_count": 25,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1414,15 +1754,6 @@
|
||||
"y_pred_rf = rnd_clf.predict(X_test)\n",
|
||||
"np.sum(y_pred == y_pred_rf) / len(y_pred)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<!-- !split -->\n",
|
||||
"## Boosting and more\n",
|
||||
"More material to come here."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {},
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,57 @@
|
||||
digraph Tree {
|
||||
node [shape=box, style="filled, rounded", color="black", fontname=helvetica] ;
|
||||
edge [fontname=helvetica] ;
|
||||
0 [label="worst perimeter <= 106.05\ngini = 0.465\nsamples = 426\nvalue = [[269, 157]\n[157, 269]]", fillcolor="#e5813908"] ;
|
||||
1 [label="worst concave points <= 0.159\ngini = 0.067\nsamples = 259\nvalue = [[250, 9]\n[9, 250]]", fillcolor="#e58139db"] ;
|
||||
0 -> 1 [labeldistance=2.5, labelangle=45, headlabel="True"] ;
|
||||
2 [label="worst concave points <= 0.135\ngini = 0.031\nsamples = 253\nvalue = [[249, 4]\n[4, 249]]", fillcolor="#e58139ee"] ;
|
||||
1 -> 2 ;
|
||||
3 [label="radius error <= 0.643\ngini = 0.008\nsamples = 242\nvalue = [[241, 1]\n[1, 241]]", fillcolor="#e58139fb"] ;
|
||||
2 -> 3 ;
|
||||
4 [label="gini = 0.0\nsamples = 239\nvalue = [[239, 0]\n[0, 239]]", fillcolor="#e58139ff"] ;
|
||||
3 -> 4 ;
|
||||
5 [label="worst symmetry <= 0.208\ngini = 0.444\nsamples = 3\nvalue = [[2, 1]\n[1, 2]]", fillcolor="#e5813913"] ;
|
||||
3 -> 5 ;
|
||||
6 [label="gini = 0.0\nsamples = 1\nvalue = [[0, 1]\n[1, 0]]", fillcolor="#e58139ff"] ;
|
||||
5 -> 6 ;
|
||||
7 [label="gini = 0.0\nsamples = 2\nvalue = [[2, 0]\n[0, 2]]", fillcolor="#e58139ff"] ;
|
||||
5 -> 7 ;
|
||||
8 [label="worst texture <= 29.455\ngini = 0.397\nsamples = 11\nvalue = [[8, 3]\n[3, 8]]", fillcolor="#e581392c"] ;
|
||||
2 -> 8 ;
|
||||
9 [label="gini = 0.0\nsamples = 8\nvalue = [[8, 0]\n[0, 8]]", fillcolor="#e58139ff"] ;
|
||||
8 -> 9 ;
|
||||
10 [label="gini = 0.0\nsamples = 3\nvalue = [[0, 3]\n[3, 0]]", fillcolor="#e58139ff"] ;
|
||||
8 -> 10 ;
|
||||
11 [label="mean texture <= 16.22\ngini = 0.278\nsamples = 6\nvalue = [[1, 5]\n[5, 1]]", fillcolor="#e581396b"] ;
|
||||
1 -> 11 ;
|
||||
12 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
11 -> 12 ;
|
||||
13 [label="gini = 0.0\nsamples = 5\nvalue = [[0, 5]\n[5, 0]]", fillcolor="#e58139ff"] ;
|
||||
11 -> 13 ;
|
||||
14 [label="worst texture <= 20.645\ngini = 0.202\nsamples = 167\nvalue = [[19, 148]\n[148, 19]]", fillcolor="#e5813994"] ;
|
||||
0 -> 14 [labeldistance=2.5, labelangle=-45, headlabel="False"] ;
|
||||
15 [label="worst radius <= 17.74\ngini = 0.375\nsamples = 16\nvalue = [[12, 4]\n[4, 12]]", fillcolor="#e5813938"] ;
|
||||
14 -> 15 ;
|
||||
16 [label="gini = 0.0\nsamples = 11\nvalue = [[11, 0]\n[0, 11]]", fillcolor="#e58139ff"] ;
|
||||
15 -> 16 ;
|
||||
17 [label="mean texture <= 13.745\ngini = 0.32\nsamples = 5\nvalue = [[1, 4]\n[4, 1]]", fillcolor="#e5813955"] ;
|
||||
15 -> 17 ;
|
||||
18 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
17 -> 18 ;
|
||||
19 [label="gini = 0.0\nsamples = 4\nvalue = [[0, 4]\n[4, 0]]", fillcolor="#e58139ff"] ;
|
||||
17 -> 19 ;
|
||||
20 [label="mean concave points <= 0.049\ngini = 0.088\nsamples = 151\nvalue = [[7, 144]\n[144, 7]]", fillcolor="#e58139d0"] ;
|
||||
14 -> 20 ;
|
||||
21 [label="concave points error <= 0.01\ngini = 0.48\nsamples = 15\nvalue = [[6, 9]\n[9, 6]]", fillcolor="#e5813900"] ;
|
||||
20 -> 21 ;
|
||||
22 [label="gini = 0.0\nsamples = 9\nvalue = [[0, 9]\n[9, 0]]", fillcolor="#e58139ff"] ;
|
||||
21 -> 22 ;
|
||||
23 [label="gini = 0.0\nsamples = 6\nvalue = [[6, 0]\n[0, 6]]", fillcolor="#e58139ff"] ;
|
||||
21 -> 23 ;
|
||||
24 [label="worst smoothness <= 0.096\ngini = 0.015\nsamples = 136\nvalue = [[1, 135]\n[135, 1]]", fillcolor="#e58139f7"] ;
|
||||
20 -> 24 ;
|
||||
25 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
24 -> 25 ;
|
||||
26 [label="gini = 0.0\nsamples = 135\nvalue = [[0, 135]\n[135, 0]]", fillcolor="#e58139ff"] ;
|
||||
24 -> 26 ;
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 216 KiB |
@@ -400,67 +400,179 @@ s = -\sum_{k=1}^K p_{mk}\log{p_{mk}}.
|
||||
|
||||
|
||||
!split
|
||||
===== The CART (Classification and Regression Tree) algorithm =====
|
||||
===== Visualizing the Tree, Classification =====
|
||||
!bc pycod
|
||||
import os
|
||||
from sklearn.datasets import load_breast_cancer
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.metrics import confusion_matrix
|
||||
from sklearn.tree import export_graphviz
|
||||
|
||||
The above functions (gini, entropy and misclassification error) are important components of the so-called CART algorithm. We will discuss this algorithm first before we move on to the information gain algorithm ID3.
|
||||
from IPython.display import Image
|
||||
from pydot import graph_from_dot_data
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
|
||||
cancer = load_breast_cancer()
|
||||
X = pd.DataFrame(cancer.data, columns=cancer.feature_names)
|
||||
print(X)
|
||||
y = pd.Categorical.from_codes(cancer.target, cancer.target_names)
|
||||
y = pd.get_dummies(y)
|
||||
print(y)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)
|
||||
tree_clf = DecisionTreeClassifier(max_depth=5)
|
||||
tree_clf.fit(X_train, y_train)
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file="DataFiles/cancer.dot",
|
||||
feature_names=cancer.feature_names,
|
||||
class_names=cancer.target_names,
|
||||
rounded=True,
|
||||
filled=True
|
||||
)
|
||||
cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
|
||||
os.system(cmd)
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Visualizing the Tree, The Moons =====
|
||||
!bc pycod
|
||||
# Common imports
|
||||
import numpy as np
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.datasets import make_moons
|
||||
from sklearn.tree import export_graphviz
|
||||
from pydot import graph_from_dot_data
|
||||
import pandas as pd
|
||||
import os
|
||||
|
||||
np.random.seed(42)
|
||||
X, y = make_moons(n_samples=100, noise=0.25, random_state=53)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)
|
||||
tree_clf = DecisionTreeClassifier(max_depth=5)
|
||||
tree_clf.fit(X_train, y_train)
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file="DataFiles/moons.dot",
|
||||
rounded=True,
|
||||
filled=True
|
||||
)
|
||||
cmd = 'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png'
|
||||
os.system(cmd)
|
||||
!ec
|
||||
|
||||
|
||||
!split
|
||||
===== Computing the Gini index =====
|
||||
|
||||
The example we will look at is a classical one in many Machine
|
||||
Learning applications. Based on various meteorological features, we
|
||||
have several so-called attributes which decide whether we at the end
|
||||
will do some outdoor activity like skiing, going for a bike ride etc
|
||||
etc. The table here contains the feautures _outlook_, _temperature_,
|
||||
_humidity_ and _wind_. The target or output is whether we ride
|
||||
(True=1) or whether we do something else that day (False=0). The
|
||||
attributes for each feature are then sunny, overcast and rain for the
|
||||
outlook, hot, cold and mild for temperature, high and normal for
|
||||
humidity and weak and strong for wind.
|
||||
|
||||
The table here summarizes the various attributes and
|
||||
|-------------------------------------------|
|
||||
|Day| Outlook |Temperature | Humidity | Wind | Ride|
|
||||
|-------------------------------------------|
|
||||
|1 | Sunny | Hot | High | Weak | 0 |
|
||||
|2 | Sunny | Hot | High | Strong | 1 |
|
||||
|3 | Overcast | Hot | High | Weak | 1 |
|
||||
|4 | Rain | Mild | High | Weak | 1 |
|
||||
|5 | Rain | Cool | Normal | Weak | 1 |
|
||||
|6 | Rain | Cool | Normal | Strong | 0 |
|
||||
|7 | Overcast | Cool | Normal | Strong | 1 |
|
||||
|8 | Sunny | Mild | High | Weak | 0 |
|
||||
|9 | Sunny | Cool | Normal | Weak | 1 |
|
||||
|10 | Rain | Mild | Normal | Weak | 1 |
|
||||
|11 | Sunny | Mild | Normal | Strong | 1 |
|
||||
|12 | Overcast | Mild | High | Strong | 1 |
|
||||
|13 | Overcast | Hot | Normal | Weak | 1 |
|
||||
|14 | Rain | Mild | High | Strong | 0 |
|
||||
|-------------------------------------------|
|
||||
|
||||
|
||||
!split
|
||||
===== Simple Python Code to read in Data =====
|
||||
|
||||
!bc pycod
|
||||
# Common imports
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.preprocessing import StandardScaler, OneHotEncoder
|
||||
from sklearn.compose import ColumnTransformer
|
||||
from IPython.display import Image
|
||||
from pydot import graph_from_dot_data
|
||||
import os
|
||||
|
||||
# Where to save the figures and data files
|
||||
PROJECT_ROOT_DIR = "Results"
|
||||
FIGURE_ID = "Results/FigureFiles"
|
||||
DATA_ID = "DataFiles/"
|
||||
|
||||
if not os.path.exists(PROJECT_ROOT_DIR):
|
||||
os.mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
if not os.path.exists(FIGURE_ID):
|
||||
os.makedirs(FIGURE_ID)
|
||||
|
||||
if not os.path.exists(DATA_ID):
|
||||
os.makedirs(DATA_ID)
|
||||
|
||||
def image_path(fig_id):
|
||||
return os.path.join(FIGURE_ID, fig_id)
|
||||
|
||||
def data_path(dat_id):
|
||||
return os.path.join(DATA_ID, dat_id)
|
||||
|
||||
def save_fig(fig_id):
|
||||
plt.savefig(image_path(fig_id) + ".png", format='png')
|
||||
|
||||
infile = open(data_path("ride.csv"),'r')
|
||||
|
||||
# Read the experimental data with Pandas
|
||||
from IPython.display import display
|
||||
ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
|
||||
ridedata = pd.DataFrame(ridedata)
|
||||
display(ridedata)
|
||||
# Features and targets
|
||||
X = ridedata.loc[:, ridedata.columns != 'Ride'].values
|
||||
display(X)
|
||||
y = ridedata.loc[:, ridedata.columns == 'Ride'].values
|
||||
display(y)
|
||||
# Categorical variables to one-hot's
|
||||
onehotencoder = OneHotEncoder(categories="auto")
|
||||
|
||||
X = ColumnTransformer([("", onehotencoder)]).fit_transform(X)
|
||||
y.shape
|
||||
|
||||
display(X)
|
||||
display(y)
|
||||
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Computing the Gini Factor =====
|
||||
|
||||
The above functions (gini, entropy and misclassification error) are
|
||||
important components of the so-called CART algorithm. We will discuss
|
||||
this algorithm below after we have discussed the information gain
|
||||
algorithm ID3.
|
||||
|
||||
In the example here we have converted all our attributes into numerical values $0,1,2$ etc.
|
||||
|
||||
!bc pycod
|
||||
from random import seed
|
||||
from random import randrange
|
||||
from csv import reader
|
||||
|
||||
# Load a CSV file
|
||||
def load_csv(filename):
|
||||
file = open(filename, "rb")
|
||||
lines = reader(file)
|
||||
dataset = list(lines)
|
||||
return dataset
|
||||
|
||||
# Convert string column to float
|
||||
def str_column_to_float(dataset, column):
|
||||
for row in dataset:
|
||||
row[column] = float(row[column].strip())
|
||||
|
||||
# Split a dataset into k folds
|
||||
def cross_validation_split(dataset, n_folds):
|
||||
dataset_split = list()
|
||||
dataset_copy = list(dataset)
|
||||
fold_size = int(len(dataset) / n_folds)
|
||||
for i in range(n_folds):
|
||||
fold = list()
|
||||
while len(fold) < fold_size:
|
||||
index = randrange(len(dataset_copy))
|
||||
fold.append(dataset_copy.pop(index))
|
||||
dataset_split.append(fold)
|
||||
return dataset_split
|
||||
|
||||
# Calculate accuracy percentage
|
||||
def accuracy_metric(actual, predicted):
|
||||
correct = 0
|
||||
for i in range(len(actual)):
|
||||
if actual[i] == predicted[i]:
|
||||
correct += 1
|
||||
return correct / float(len(actual)) * 100.0
|
||||
|
||||
# Evaluate an algorithm using a cross validation split
|
||||
def evaluate_algorithm(dataset, algorithm, n_folds, *args):
|
||||
folds = cross_validation_split(dataset, n_folds)
|
||||
scores = list()
|
||||
for fold in folds:
|
||||
train_set = list(folds)
|
||||
train_set.remove(fold)
|
||||
train_set = sum(train_set, [])
|
||||
test_set = list()
|
||||
for row in fold:
|
||||
row_copy = list(row)
|
||||
test_set.append(row_copy)
|
||||
row_copy[-1] = None
|
||||
predicted = algorithm(train_set, test_set, *args)
|
||||
actual = [row[-1] for row in fold]
|
||||
accuracy = accuracy_metric(actual, predicted)
|
||||
scores.append(accuracy)
|
||||
return scores
|
||||
|
||||
# Split a dataset based on an attribute and an attribute value
|
||||
def test_split(index, value, dataset):
|
||||
left, right = list(), list()
|
||||
@@ -490,7 +602,7 @@ def gini_index(groups, classes):
|
||||
# weight the group score by its relative size
|
||||
gini += (1.0 - score) * (size / n_instances)
|
||||
return gini
|
||||
|
||||
|
||||
# Select the best split point for a dataset
|
||||
def get_split(dataset):
|
||||
class_values = list(set(row[-1] for row in dataset))
|
||||
@@ -499,84 +611,28 @@ def get_split(dataset):
|
||||
for row in dataset:
|
||||
groups = test_split(index, row[index], dataset)
|
||||
gini = gini_index(groups, class_values)
|
||||
print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini))
|
||||
if gini < b_score:
|
||||
b_index, b_value, b_score, b_groups = index, row[index], gini, groups
|
||||
return {'index':b_index, 'value':b_value, 'groups':b_groups}
|
||||
|
||||
# Create a terminal node value
|
||||
def to_terminal(group):
|
||||
outcomes = [row[-1] for row in group]
|
||||
return max(set(outcomes), key=outcomes.count)
|
||||
|
||||
# Create child splits for a node or make terminal
|
||||
def split(node, max_depth, min_size, depth):
|
||||
left, right = node['groups']
|
||||
del(node['groups'])
|
||||
# check for a no split
|
||||
if not left or not right:
|
||||
node['left'] = node['right'] = to_terminal(left + right)
|
||||
return
|
||||
# check for max depth
|
||||
if depth >= max_depth:
|
||||
node['left'], node['right'] = to_terminal(left), to_terminal(right)
|
||||
return
|
||||
# process left child
|
||||
if len(left) <= min_size:
|
||||
node['left'] = to_terminal(left)
|
||||
else:
|
||||
node['left'] = get_split(left)
|
||||
split(node['left'], max_depth, min_size, depth+1)
|
||||
# process right child
|
||||
if len(right) <= min_size:
|
||||
node['right'] = to_terminal(right)
|
||||
else:
|
||||
node['right'] = get_split(right)
|
||||
split(node['right'], max_depth, min_size, depth+1)
|
||||
|
||||
# Build a decision tree
|
||||
def build_tree(train, max_depth, min_size):
|
||||
root = get_split(train)
|
||||
split(root, max_depth, min_size, 1)
|
||||
return root
|
||||
|
||||
# Make a prediction with a decision tree
|
||||
def predict(node, row):
|
||||
if row[node['index']] < node['value']:
|
||||
if isinstance(node['left'], dict):
|
||||
return predict(node['left'], row)
|
||||
else:
|
||||
return node['left']
|
||||
else:
|
||||
if isinstance(node['right'], dict):
|
||||
return predict(node['right'], row)
|
||||
else:
|
||||
return node['right']
|
||||
|
||||
# Classification and Regression Tree Algorithm
|
||||
def decision_tree(train, test, max_depth, min_size):
|
||||
tree = build_tree(train, max_depth, min_size)
|
||||
predictions = list()
|
||||
for row in test:
|
||||
prediction = predict(tree, row)
|
||||
predictions.append(prediction)
|
||||
return(predictions)
|
||||
|
||||
# Test CART
|
||||
seed(1)
|
||||
# load and prepare data
|
||||
filename = 'DataFiles/rideclass.csv'
|
||||
dataset = load_csv(filename)
|
||||
# convert string attributes to integers
|
||||
for i in range(len(dataset[0])):
|
||||
str_column_to_float(dataset, i)
|
||||
# evaluate algorithm
|
||||
n_folds = 5
|
||||
max_depth = 5
|
||||
min_size = 10
|
||||
scores = evaluate_algorithm(dataset, decision_tree, n_folds, max_depth, min_size)
|
||||
print('Scores: %s' % scores)
|
||||
print('Mean Accuracy: %.3f%%' % (sum(scores)/float(len(scores))))
|
||||
dataset = [[0,0,0,0,0],
|
||||
[0,0,0,1,1],
|
||||
[1,0,0,0,1],
|
||||
[2,1,0,0,1],
|
||||
[2,2,1,0,1],
|
||||
[2,2,1,1,0],
|
||||
[1,2,1,1,1],
|
||||
[0,1,0,0,0],
|
||||
[0,2,1,0,1],
|
||||
[2,1,1,0,1],
|
||||
[0,1,1,1,1],
|
||||
[1,1,0,1,1],
|
||||
[1,0,1,0,1],
|
||||
[2,1,0,1,0]]
|
||||
|
||||
split = get_split(dataset)
|
||||
print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
|
||||
!ec
|
||||
|
||||
!split
|
||||
@@ -609,10 +665,197 @@ attributes at each step while growing the tree.
|
||||
!split
|
||||
===== Implementing the ID3 Algorithm =====
|
||||
|
||||
_more text to come here_, material presented during lecture Friday Oct 25.
|
||||
import re
|
||||
import math
|
||||
from collections import deque
|
||||
|
||||
# x is examples in training set
|
||||
# y is set of attributes
|
||||
# label is target attributes
|
||||
# Node is a class which has properties values, childs, and next
|
||||
# root is top node in the decision tree
|
||||
|
||||
class Node(object):
|
||||
def __init__(self):
|
||||
self.value = None
|
||||
self.next = None
|
||||
self.childs = None
|
||||
|
||||
# Simple class of Decision Tree
|
||||
# Aimed for who want to learn Decision Tree, so it is not optimized
|
||||
class DecisionTree(object):
|
||||
def __init__(self, sample, attributes, labels):
|
||||
self.sample = sample
|
||||
self.attributes = attributes
|
||||
self.labels = labels
|
||||
self.labelCodes = None
|
||||
self.labelCodesCount = None
|
||||
self.initLabelCodes()
|
||||
# print(self.labelCodes)
|
||||
self.root = None
|
||||
self.entropy = self.getEntropy([x for x in range(len(self.labels))])
|
||||
|
||||
def initLabelCodes(self):
|
||||
self.labelCodes = []
|
||||
self.labelCodesCount = []
|
||||
for l in self.labels:
|
||||
if l not in self.labelCodes:
|
||||
self.labelCodes.append(l)
|
||||
self.labelCodesCount.append(0)
|
||||
self.labelCodesCount[self.labelCodes.index(l)] += 1
|
||||
|
||||
def getLabelCodeId(self, sampleId):
|
||||
return self.labelCodes.index(self.labels[sampleId])
|
||||
|
||||
def getAttributeValues(self, sampleIds, attributeId):
|
||||
vals = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in vals:
|
||||
vals.append(val)
|
||||
# print(vals)
|
||||
return vals
|
||||
|
||||
def getEntropy(self, sampleIds):
|
||||
entropy = 0
|
||||
labelCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCount[self.getLabelCodeId(sid)] += 1
|
||||
# print("-ge", labelCount)
|
||||
for lv in labelCount:
|
||||
# print(lv)
|
||||
if lv != 0:
|
||||
entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
|
||||
else:
|
||||
entropy += 0
|
||||
return entropy
|
||||
|
||||
def getDominantLabel(self, sampleIds):
|
||||
labelCodesCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
|
||||
return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
|
||||
|
||||
def getInformationGain(self, sampleIds, attributeId):
|
||||
gain = self.getEntropy(sampleIds)
|
||||
attributeVals = []
|
||||
attributeValsCount = []
|
||||
attributeValsIds = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in attributeVals:
|
||||
attributeVals.append(val)
|
||||
attributeValsCount.append(0)
|
||||
attributeValsIds.append([])
|
||||
vid = attributeVals.index(val)
|
||||
attributeValsCount[vid] += 1
|
||||
attributeValsIds[vid].append(sid)
|
||||
# print("-gig", self.attributes[attributeId])
|
||||
for vc, vids in zip(attributeValsCount, attributeValsIds):
|
||||
# print("-gig", vids)
|
||||
gain -= vc/len(sampleIds) * self.getEntropy(vids)
|
||||
return gain
|
||||
|
||||
def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
|
||||
attributesEntropy = [0] * len(attributeIds)
|
||||
for i, attId in zip(range(len(attributeIds)), attributeIds):
|
||||
attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
|
||||
maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
|
||||
return self.attributes[maxId], maxId
|
||||
|
||||
def isSingleLabeled(self, sampleIds):
|
||||
label = self.labels[sampleIds[0]]
|
||||
for sid in sampleIds:
|
||||
if self.labels[sid] != label:
|
||||
return False
|
||||
return True
|
||||
|
||||
def getLabel(self, sampleId):
|
||||
return self.labels[sampleId]
|
||||
|
||||
def id3(self):
|
||||
sampleIds = [x for x in range(len(self.sample))]
|
||||
attributeIds = [x for x in range(len(self.attributes))]
|
||||
self.root = self.id3Recv(sampleIds, attributeIds, self.root)
|
||||
|
||||
def id3Recv(self, sampleIds, attributeIds, root):
|
||||
root = Node() # Initialize current root
|
||||
if self.isSingleLabeled(sampleIds):
|
||||
root.value = self.labels[sampleIds[0]]
|
||||
return root
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) == 0:
|
||||
root.value = self.getDominantLabel(sampleIds)
|
||||
return root
|
||||
bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
|
||||
sampleIds, attributeIds)
|
||||
# print(bestAttrName)
|
||||
root.value = bestAttrName
|
||||
root.childs = [] # Create list of children
|
||||
for value in self.getAttributeValues(sampleIds, bestAttrId):
|
||||
# print(value)
|
||||
child = Node()
|
||||
child.value = value
|
||||
root.childs.append(child) # Append new child node to current
|
||||
# root
|
||||
childSampleIds = []
|
||||
for sid in sampleIds:
|
||||
if self.sample[sid][bestAttrId] == value:
|
||||
childSampleIds.append(sid)
|
||||
if len(childSampleIds) == 0:
|
||||
child.next = self.getDominantLabel(sampleIds)
|
||||
else:
|
||||
# print(bestAttrName, bestAttrId)
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) > 0 and bestAttrId in attributeIds:
|
||||
toRemove = attributeIds.index(bestAttrId)
|
||||
attributeIds.pop(toRemove)
|
||||
child.next = self.id3Recv(
|
||||
childSampleIds, attributeIds, child.next)
|
||||
return root
|
||||
|
||||
def printTree(self):
|
||||
if self.root:
|
||||
roots = deque()
|
||||
roots.append(self.root)
|
||||
while len(roots) > 0:
|
||||
root = roots.popleft()
|
||||
print(root.value)
|
||||
if root.childs:
|
||||
for child in root.childs:
|
||||
print('({})'.format(child.value))
|
||||
roots.append(child.next)
|
||||
elif root.next:
|
||||
print(root.next)
|
||||
|
||||
|
||||
def test():
|
||||
f = open('DataFiles/rideclass.csv')
|
||||
attributes = f.readline().split(',')
|
||||
attributes = attributes[1:len(attributes)-1]
|
||||
print(attributes)
|
||||
sample = f.readlines()
|
||||
f.close()
|
||||
for i in range(len(sample)):
|
||||
sample[i] = re.sub('\d+,', '', sample[i])
|
||||
sample[i] = sample[i].strip().split(',')
|
||||
labels = []
|
||||
for s in sample:
|
||||
labels.append(s.pop())
|
||||
# print(sample)
|
||||
# print(labels)
|
||||
decisionTree = DecisionTree(sample, attributes, labels)
|
||||
print("System entropy {}".format(decisionTree.entropy))
|
||||
decisionTree.id3()
|
||||
decisionTree.printTree()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
|
||||
|
||||
!split
|
||||
===== Cancer Data again now with Decision Trees =====
|
||||
===== Cancer Data again now with Decision Trees and other Methods =====
|
||||
!bc pycod
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
@@ -736,7 +979,7 @@ np.random.seed(6)
|
||||
Xs = np.random.rand(100, 2) - 0.5
|
||||
ys = (Xs[:, 0] > 0).astype(np.float32) * 2
|
||||
|
||||
angle = np.pi / 4
|
||||
angle = np.pi/4
|
||||
rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
|
||||
Xsr = Xs.dot(rotation_matrix)
|
||||
|
||||
@@ -889,6 +1132,9 @@ general-purpose procedure for reducing the variance of a statistical
|
||||
learning method.
|
||||
|
||||
|
||||
!split
|
||||
===== More bagging =====
|
||||
|
||||
Bagging typically results in improved accuracy
|
||||
over prediction using a single tree. Unfortunately, however, it can be
|
||||
difficult to interpret the resulting model. Recall that one of the
|
||||
@@ -929,6 +1175,56 @@ plt.show()
|
||||
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Bagging Example =====
|
||||
!bc pycod
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.datasets import make_moons
|
||||
|
||||
X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
|
||||
|
||||
from sklearn.ensemble import RandomForestClassifier
|
||||
from sklearn.ensemble import VotingClassifier
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.svm import SVC
|
||||
|
||||
log_clf = LogisticRegression(solver="liblinear", random_state=42)
|
||||
rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
|
||||
svm_clf = SVC(gamma="auto", random_state=42)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
|
||||
voting='hard')
|
||||
|
||||
voting_clf.fit(X_train, y_train)
|
||||
|
||||
from sklearn.metrics import accuracy_score
|
||||
|
||||
for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
|
||||
|
||||
log_clf = LogisticRegression(solver="liblinear", random_state=42)
|
||||
rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
|
||||
svm_clf = SVC(gamma="auto", probability=True, random_state=42)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
|
||||
voting='soft')
|
||||
voting_clf.fit(X_train, y_train)
|
||||
|
||||
from sklearn.metrics import accuracy_score
|
||||
|
||||
for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
|
||||
|
||||
!ec
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== Random forests =====
|
||||
@@ -1113,8 +1409,4 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
|
||||
!ec
|
||||
|
||||
|
||||
!split
|
||||
===== Boosting and more =====
|
||||
More material to come here.
|
||||
|
||||
|
||||
|
||||
+4
-4
@@ -10,7 +10,7 @@ edge [fontname=helvetica] ;
|
||||
2 -> 3 ;
|
||||
4 [label="gini = 0.0\nsamples = 239\nvalue = [[239, 0]\n[0, 239]]", fillcolor="#e58139ff"] ;
|
||||
3 -> 4 ;
|
||||
5 [label="symmetry error <= 0.025\ngini = 0.444\nsamples = 3\nvalue = [[2, 1]\n[1, 2]]", fillcolor="#e5813913"] ;
|
||||
5 [label="compactness error <= 0.014\ngini = 0.444\nsamples = 3\nvalue = [[2, 1]\n[1, 2]]", fillcolor="#e5813913"] ;
|
||||
3 -> 5 ;
|
||||
6 [label="gini = 0.0\nsamples = 1\nvalue = [[0, 1]\n[1, 0]]", fillcolor="#e58139ff"] ;
|
||||
5 -> 6 ;
|
||||
@@ -22,7 +22,7 @@ edge [fontname=helvetica] ;
|
||||
8 -> 9 ;
|
||||
10 [label="gini = 0.0\nsamples = 3\nvalue = [[0, 3]\n[3, 0]]", fillcolor="#e58139ff"] ;
|
||||
8 -> 10 ;
|
||||
11 [label="worst texture <= 24.785\ngini = 0.278\nsamples = 6\nvalue = [[1, 5]\n[5, 1]]", fillcolor="#e581396b"] ;
|
||||
11 [label="mean texture <= 16.22\ngini = 0.278\nsamples = 6\nvalue = [[1, 5]\n[5, 1]]", fillcolor="#e581396b"] ;
|
||||
1 -> 11 ;
|
||||
12 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
11 -> 12 ;
|
||||
@@ -30,11 +30,11 @@ edge [fontname=helvetica] ;
|
||||
11 -> 13 ;
|
||||
14 [label="worst texture <= 20.645\ngini = 0.202\nsamples = 167\nvalue = [[19, 148]\n[148, 19]]", fillcolor="#e5813994"] ;
|
||||
0 -> 14 [labeldistance=2.5, labelangle=-45, headlabel="False"] ;
|
||||
15 [label="worst perimeter <= 116.8\ngini = 0.375\nsamples = 16\nvalue = [[12, 4]\n[4, 12]]", fillcolor="#e5813938"] ;
|
||||
15 [label="worst area <= 964.4\ngini = 0.375\nsamples = 16\nvalue = [[12, 4]\n[4, 12]]", fillcolor="#e5813938"] ;
|
||||
14 -> 15 ;
|
||||
16 [label="gini = 0.0\nsamples = 11\nvalue = [[11, 0]\n[0, 11]]", fillcolor="#e58139ff"] ;
|
||||
15 -> 16 ;
|
||||
17 [label="mean concavity <= 0.06\ngini = 0.32\nsamples = 5\nvalue = [[1, 4]\n[4, 1]]", fillcolor="#e5813955"] ;
|
||||
17 [label="worst compactness <= 0.126\ngini = 0.32\nsamples = 5\nvalue = [[1, 4]\n[4, 1]]", fillcolor="#e5813955"] ;
|
||||
15 -> 17 ;
|
||||
18 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
17 -> 18 ;
|
||||
@@ -1,3 +1,4 @@
|
||||
import os
|
||||
from sklearn.datasets import load_breast_cancer
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.model_selection import train_test_split
|
||||
@@ -9,6 +10,31 @@ from pydot import graph_from_dot_data
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
|
||||
# Where to save the figures and data files
|
||||
PROJECT_ROOT_DIR = "Results"
|
||||
FIGURE_ID = "Results/FigureFiles"
|
||||
DATA_ID = "DataFiles/"
|
||||
|
||||
if not os.path.exists(PROJECT_ROOT_DIR):
|
||||
os.mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
if not os.path.exists(FIGURE_ID):
|
||||
os.makedirs(FIGURE_ID)
|
||||
|
||||
if not os.path.exists(DATA_ID):
|
||||
os.makedirs(DATA_ID)
|
||||
|
||||
def image_path(fig_id):
|
||||
return os.path.join(FIGURE_ID, fig_id)
|
||||
|
||||
def data_path(dat_id):
|
||||
return os.path.join(DATA_ID, dat_id)
|
||||
|
||||
def save_fig(fig_id):
|
||||
plt.savefig(image_path(fig_id) + ".png", format='png')
|
||||
|
||||
|
||||
cancer = load_breast_cancer()
|
||||
X = pd.DataFrame(cancer.data, columns=cancer.feature_names)
|
||||
print(X)
|
||||
@@ -21,10 +47,11 @@ tree_clf.fit(X_train, y_train)
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file="cancer.dot",
|
||||
out_file="DataFiles/cancer.dot",
|
||||
feature_names=cancer.feature_names,
|
||||
class_names=cancer.target_names,
|
||||
rounded=True,
|
||||
filled=True
|
||||
)
|
||||
|
||||
cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
|
||||
os.system(cmd)
|
||||
|
||||
@@ -41,6 +41,7 @@ def get_split(dataset):
|
||||
b_index, b_value, b_score, b_groups = index, row[index], gini, groups
|
||||
return {'index':b_index, 'value':b_value, 'groups':b_groups}
|
||||
|
||||
"""
|
||||
dataset = [[2.771244718,1.784783929,0],
|
||||
[1.728571309,1.169761413,0],
|
||||
[3.678319846,2.81281357,0],
|
||||
@@ -51,5 +52,23 @@ dataset = [[2.771244718,1.784783929,0],
|
||||
[7.444542326,0.476683375,1],
|
||||
[10.12493903,3.234550982,1],
|
||||
[6.642287351,3.319983761,1]]
|
||||
"""
|
||||
dataset = [[0,0,0,0,0],
|
||||
[0,0,0,1,1],
|
||||
[1,0,0,0,1],
|
||||
[2,1,0,0,1],
|
||||
[2,2,1,0,1],
|
||||
[2,2,1,1,0],
|
||||
[1,2,1,1,1],
|
||||
[0,1,0,0,0],
|
||||
[0,2,1,0,1],
|
||||
[2,1,1,0,1],
|
||||
[0,1,1,1,1],
|
||||
[1,1,0,1,1],
|
||||
[1,0,1,0,1],
|
||||
[2,1,0,1,0]]
|
||||
|
||||
split = get_split(dataset)
|
||||
print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user