From b39cfc27e9f2fb7c9444335b7cea0f3cda09987b Mon Sep 17 00:00:00 2001 From: mhjensen Date: Tue, 29 Oct 2019 20:40:30 +0100 Subject: [PATCH] building up codes --- .../html/._DecisionTrees-bs000.html | 77 ++--- .../html/._DecisionTrees-bs001.html | 75 ++--- .../html/._DecisionTrees-bs002.html | 75 ++--- .../html/._DecisionTrees-bs003.html | 75 ++--- .../html/._DecisionTrees-bs004.html | 75 ++--- .../html/._DecisionTrees-bs005.html | 75 ++--- .../html/._DecisionTrees-bs006.html | 75 ++--- .../html/._DecisionTrees-bs007.html | 75 ++--- .../html/._DecisionTrees-bs008.html | 75 ++--- .../html/._DecisionTrees-bs009.html | 75 ++--- .../html/._DecisionTrees-bs010.html | 75 ++--- .../html/._DecisionTrees-bs011.html | 75 ++--- .../html/._DecisionTrees-bs012.html | 75 ++--- .../html/._DecisionTrees-bs013.html | 75 ++--- .../html/._DecisionTrees-bs014.html | 75 ++--- .../html/._DecisionTrees-bs015.html | 75 ++--- .../html/._DecisionTrees-bs016.html | 278 ++++++++++++++---- .../html/._DecisionTrees-bs017.html | 107 ++++--- .../html/._DecisionTrees-bs018.html | 122 +++----- .../html/._DecisionTrees-bs019.html | 172 +++++------ .../html/._DecisionTrees-bs020.html | 147 ++++++--- .../html/._DecisionTrees-bs021.html | 107 ++++--- .../html/._DecisionTrees-bs022.html | 157 ++++------ .../html/._DecisionTrees-bs023.html | 159 +++++++--- .../html/._DecisionTrees-bs024.html | 93 +++--- .../html/._DecisionTrees-bs025.html | 122 ++++---- .../html/._DecisionTrees-bs026.html | 129 ++++---- .../html/._DecisionTrees-bs027.html | 128 ++++---- .../html/._DecisionTrees-bs028.html | 128 +++++--- .../html/._DecisionTrees-bs029.html | 136 ++++----- .../html/._DecisionTrees-bs030.html | 149 +++++----- .../html/._DecisionTrees-bs031.html | 135 ++++++--- .../DecisionTrees/html/DecisionTrees-bs.html | 77 ++--- .../html/DecisionTrees-reveal.html | 220 ++++++++++++-- .../html/DecisionTrees-solarized.html | 257 +++++++++++++--- doc/pub/DecisionTrees/html/DecisionTrees.html | 257 +++++++++++++--- .../DecisionTrees/ipynb/DecisionTrees.ipynb | 230 +++++++++++++-- .../ipynb/ipynb-DecisionTrees-src.tar.gz | Bin 194 -> 194 bytes .../pdf/DecisionTrees-minted.pdf | Bin 265673 -> 266231 bytes doc/src/DecisionTrees/DecisionTrees.do.txt | 180 ++++++++++++ doc/src/DecisionTrees/cart.py~ | 172 ----------- doc/src/DecisionTrees/decisiontree.py~ | 187 ------------ doc/src/DecisionTrees/dtcancer.py~ | 29 -- doc/src/DecisionTrees/read.py~ | 80 ----- 44 files changed, 2978 insertions(+), 2182 deletions(-) delete mode 100644 doc/src/DecisionTrees/cart.py~ delete mode 100644 doc/src/DecisionTrees/decisiontree.py~ delete mode 100644 doc/src/DecisionTrees/dtcancer.py~ delete mode 100644 doc/src/DecisionTrees/read.py~ diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs000.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs000.html index ceb0e0248..8136086bf 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs000.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs000.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -190,7 +195,7 @@ MathJax.Hub.Config({
    [2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University

    -

    Oct 26, 2019

    +

    Oct 29, 2019


    @@ -214,7 +219,7 @@ MathJax.Hub.Config({

  • 9
  • 10
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs001.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs001.html index 3b30bddfe..017dbf0d2 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs001.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs001.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -219,7 +224,7 @@ given some assumptions, make predictions about the target feature value
  • 10
  • 11
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs002.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs002.html index cabb06ae8..dd41cd135 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs002.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs002.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -194,7 +199,7 @@ In the figure here we present a decision tree obtained from a classification pro
  • 11
  • 12
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs003.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs003.html index 730fc99c9..827610f21 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs003.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs003.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -195,7 +200,7 @@ In the figure we present a decision tree obtained from a simple regression prob
  • 12
  • 13
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs004.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs004.html index 05d7cc3c5..f9696d635 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs004.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs004.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -206,7 +211,7 @@ node.
  • 13
  • 14
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs005.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs005.html index 43bf8f88d..fe313790f 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs005.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs005.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -207,7 +212,7 @@ Then we are essentially done!
  • 14
  • 15
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs006.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs006.html index d02772a03..5ea3aa965 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs006.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs006.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -286,7 +291,7 @@ plt.show()
  • 15
  • 16
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs007.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs007.html index b4f46b182..45d860328 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs007.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs007.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -219,7 +224,7 @@ within box \( j \).
  • 16
  • 17
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs008.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs008.html index 5726c850b..c0391d238 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs008.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs008.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -211,7 +216,7 @@ better tree in some future step.
  • 17
  • 18
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs009.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs009.html index 7c6e70eda..29a2bf72e 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs009.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs009.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -244,7 +249,7 @@ region contains more than five observations.
  • 18
  • 19
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs010.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs010.html index 2831c11d5..9b03e5c6f 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs010.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs010.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -213,7 +218,7 @@ parameter \( \alpha \).
  • 19
  • 20
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs011.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs011.html index c14cd1bd9..f6e230314 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs011.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs011.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -225,7 +230,7 @@ subtree corresponding to \( \alpha \).
  • 20
  • 21
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs012.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs012.html index 672b5a2fc..d660d8e01 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs012.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs012.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -221,7 +226,7 @@ MathJax.Hub.Config({
  • 21
  • 22
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs013.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs013.html index 8be931c10..fed826427 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs013.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs013.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -213,7 +218,7 @@ fall into that region.
  • 22
  • 23
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs014.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs014.html index 0c50d4fb1..16a25427e 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs014.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs014.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -218,7 +223,7 @@ than is the classification error rate.
  • 23
  • 24
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs015.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs015.html index 1bd9c6979..6a1d92ce7 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs015.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs015.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -244,7 +249,7 @@ $$
  • 24
  • 25
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs016.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs016.html index b2914688d..0486b997b 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs016.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs016.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -171,39 +176,186 @@ MathJax.Hub.Config({ -

    Entropy and the ID3 algorithm

    +

    The CART (Classification and Regression Tree) algorithm

    -ID3, learns decision trees by constructing -them topdown, beginning with the question which attribute should be tested at the root of the tree? - -

      -
    1. Each instance attribute is evaluated using a statistical test to determine how well it alone classifies the training examples.
      2. -
    2. The best attribute is selected and used as the test at the root node of the tree.
      3. -
    3. A descendant of the root node is then created for each possible value of this attribute.
      4. -
    4. Training examples are sorted to the appropriate descendant node.
      5. -
    5. The entire process is then repeated using the training examples associated with each descendant node to select the best attribute to test at that point in the tree.
      6. -
    6. This forms a greedy search for an acceptable decision tree, in which the algorithm never backtracks to reconsider earlier choices.
      7. -
    - -The ID3 algorithm selects, which attribute to test at each node in the -tree. +The above functions (gini, entropy and misclassification error) are important components of the so-called CART algorithm. We will discuss this algorithm first before we move on to the information gain algorithm ID3.

    -We would like to select the attribute that is most useful for classifying -examples. - -

    -What is a good quantitative measure of the worth of an attribute? - -

    -Information gain measures how well a given attribute separates the -training examples according to their target classification. - -

    -The ID3 algorithm uses this information gain measure to select among the candidate -attributes at each step while growing the tree. + +

    from random import seed
+from random import randrange
+from csv import reader
+ 
+# Load a CSV file
+def load_csv(filename):
+	file = open(filename, "rb")
+	lines = reader(file)
+	dataset = list(lines)
+	return dataset
+ 
+# Convert string column to float
+def str_column_to_float(dataset, column):
+	for row in dataset:
+		row[column] = float(row[column].strip())
+ 
+# Split a dataset into k folds
+def cross_validation_split(dataset, n_folds):
+	dataset_split = list()
+	dataset_copy = list(dataset)
+	fold_size = int(len(dataset) / n_folds)
+	for i in range(n_folds):
+		fold = list()
+		while len(fold) < fold_size:
+			index = randrange(len(dataset_copy))
+			fold.append(dataset_copy.pop(index))
+		dataset_split.append(fold)
+	return dataset_split
+ 
+# Calculate accuracy percentage
+def accuracy_metric(actual, predicted):
+	correct = 0
+	for i in range(len(actual)):
+		if actual[i] == predicted[i]:
+			correct += 1
+	return correct / float(len(actual)) * 100.0
+ 
+# Evaluate an algorithm using a cross validation split
+def evaluate_algorithm(dataset, algorithm, n_folds, *args):
+	folds = cross_validation_split(dataset, n_folds)
+	scores = list()
+	for fold in folds:
+		train_set = list(folds)
+		train_set.remove(fold)
+		train_set = sum(train_set, [])
+		test_set = list()
+		for row in fold:
+			row_copy = list(row)
+			test_set.append(row_copy)
+			row_copy[-1] = None
+		predicted = algorithm(train_set, test_set, *args)
+		actual = [row[-1] for row in fold]
+		accuracy = accuracy_metric(actual, predicted)
+		scores.append(accuracy)
+	return scores
+ 
+# Split a dataset based on an attribute and an attribute value
+def test_split(index, value, dataset):
+	left, right = list(), list()
+	for row in dataset:
+		if row[index] < value:
+			left.append(row)
+		else:
+			right.append(row)
+	return left, right
+ 
+# Calculate the Gini index for a split dataset
+def gini_index(groups, classes):
+	# count all samples at split point
+	n_instances = float(sum([len(group) for group in groups]))
+	# sum weighted Gini index for each group
+	gini = 0.0
+	for group in groups:
+		size = float(len(group))
+		# avoid divide by zero
+		if size == 0:
+			continue
+		score = 0.0
+		# score the group based on the score for each class
+		for class_val in classes:
+			p = [row[-1] for row in group].count(class_val) / size
+			score += p * p
+		# weight the group score by its relative size
+		gini += (1.0 - score) * (size / n_instances)
+	return gini
+ 
+# Select the best split point for a dataset
+def get_split(dataset):
+	class_values = list(set(row[-1] for row in dataset))
+	b_index, b_value, b_score, b_groups = 999, 999, 999, None
+	for index in range(len(dataset[0])-1):
+		for row in dataset:
+			groups = test_split(index, row[index], dataset)
+			gini = gini_index(groups, class_values)
+			if gini < b_score:
+				b_index, b_value, b_score, b_groups = index, row[index], gini, groups
+	return {'index':b_index, 'value':b_value, 'groups':b_groups}
+ 
+# Create a terminal node value
+def to_terminal(group):
+	outcomes = [row[-1] for row in group]
+	return max(set(outcomes), key=outcomes.count)
+ 
+# Create child splits for a node or make terminal
+def split(node, max_depth, min_size, depth):
+	left, right = node['groups']
+	del(node['groups'])
+	# check for a no split
+	if not left or not right:
+		node['left'] = node['right'] = to_terminal(left + right)
+		return
+	# check for max depth
+	if depth >= max_depth:
+		node['left'], node['right'] = to_terminal(left), to_terminal(right)
+		return
+	# process left child
+	if len(left) <= min_size:
+		node['left'] = to_terminal(left)
+	else:
+		node['left'] = get_split(left)
+		split(node['left'], max_depth, min_size, depth+1)
+	# process right child
+	if len(right) <= min_size:
+		node['right'] = to_terminal(right)
+	else:
+		node['right'] = get_split(right)
+		split(node['right'], max_depth, min_size, depth+1)
+ 
+# Build a decision tree
+def build_tree(train, max_depth, min_size):
+	root = get_split(train)
+	split(root, max_depth, min_size, 1)
+	return root
+ 
+# Make a prediction with a decision tree
+def predict(node, row):
+	if row[node['index']] < node['value']:
+		if isinstance(node['left'], dict):
+			return predict(node['left'], row)
+		else:
+			return node['left']
+	else:
+		if isinstance(node['right'], dict):
+			return predict(node['right'], row)
+		else:
+			return node['right']
+ 
+# Classification and Regression Tree Algorithm
+def decision_tree(train, test, max_depth, min_size):
+	tree = build_tree(train, max_depth, min_size)
+	predictions = list()
+	for row in test:
+		prediction = predict(tree, row)
+		predictions.append(prediction)
+	return(predictions)
+ 
+# Test CART 
+seed(1)
+# load and prepare data
+filename = 'DataFiles/rideclass.csv'
+dataset = load_csv(filename)
+# convert string attributes to integers
+for i in range(len(dataset[0])):
+	str_column_to_float(dataset, i)
+# evaluate algorithm
+n_folds = 5
+max_depth = 5
+min_size = 10
+scores = evaluate_algorithm(dataset, decision_tree, n_folds, max_depth, min_size)
+print('Scores: %s' % scores)
+print('Mean Accuracy: %.3f%%' % (sum(scores)/float(len(scores))))
+

    @@ -230,7 +382,7 @@ attributes at each step while growing the tree.

  • 25
  • 26
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs017.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs017.html index a762e5c1b..3ff2e4c07 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs017.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs017.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -171,10 +176,38 @@ MathJax.Hub.Config({ -

    Implementing the ID3 Algorithm

    +

    Entropy and the ID3 algorithm

    -more text to come here, material presented during lecture Friday Oct 25. +ID3, learns decision trees by constructing +them topdown, beginning with the question which attribute should be tested at the root of the tree? + +

      +
    1. Each instance attribute is evaluated using a statistical test to determine how well it alone classifies the training examples.
      2. +
    2. The best attribute is selected and used as the test at the root node of the tree.
      3. +
    3. A descendant of the root node is then created for each possible value of this attribute.
      4. +
    4. Training examples are sorted to the appropriate descendant node.
      5. +
    5. The entire process is then repeated using the training examples associated with each descendant node to select the best attribute to test at that point in the tree.
      6. +
    6. This forms a greedy search for an acceptable decision tree, in which the algorithm never backtracks to reconsider earlier choices.
      7. +
    + +The ID3 algorithm selects, which attribute to test at each node in the +tree. + +

    +We would like to select the attribute that is most useful for classifying +examples. + +

    +What is a good quantitative measure of the worth of an attribute? + +

    +Information gain measures how well a given attribute separates the +training examples according to their target classification. + +

    +The ID3 algorithm uses this information gain measure to select among the candidate +attributes at each step while growing the tree.

    @@ -202,7 +235,7 @@ MathJax.Hub.Config({

  • 26
  • 27
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs018.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs018.html index 8a5f891fe..0b61ea913 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs018.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs018.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -171,52 +176,11 @@ MathJax.Hub.Config({ -

    Cancer Data again now with Decision Trees

    +

    Implementing the ID3 Algorithm

    +

    +more text to come here, material presented during lecture Friday Oct 25. - -

    import matplotlib.pyplot as plt
-import numpy as np
-from sklearn.model_selection import  train_test_split 
-from sklearn.datasets import load_breast_cancer
-from sklearn.svm import SVC
-from sklearn.linear_model import LogisticRegression
-from sklearn.tree import DecisionTreeClassifier
-
-# Load the data
-cancer = load_breast_cancer()
-
-X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-print(X_train.shape)
-print(X_test.shape)
-# Logistic Regression
-logreg = LogisticRegression(solver='lbfgs')
-logreg.fit(X_train, y_train)
-print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
-# Support vector machine
-svm = SVC(gamma='auto', C=100)
-svm.fit(X_train, y_train)
-print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
-# Decision Trees
-deep_tree_clf = DecisionTreeClassifier(max_depth=None)
-deep_tree_clf.fit(X_train, y_train)
-print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
-#now scale the data
-from sklearn.preprocessing import StandardScaler
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-# Logistic Regression
-logreg.fit(X_train_scaled, y_train)
-print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Support Vector Machine
-svm.fit(X_train_scaled, y_train)
-print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Decision Trees
-deep_tree_clf.fit(X_train_scaled, y_train)
-print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
-

    @@ -243,7 +207,7 @@ deep_tree_clf.fit(X_train_scaled, y_train)

  • 27
  • 28
  • ...
    • -
  • 33
    • +
  • 34
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs019.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs019.html index 021d0b636..c9605d53a 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs019.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs019.html @@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source ('A Classification Tree', 2, None, '___sec12'), ('Growing a classification tree', 2, None, '___sec13'), ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Entropy and the ID3 algorithm', 2, None, '___sec15'), - ('Implementing the ID3 Algorithm', 2, None, '___sec16'), + ('The CART (Classification and Regression Tree) algorithm', + 2, + None, + '___sec15'), + ('Entropy and the ID3 algorithm', 2, None, '___sec16'), + ('Implementing the ID3 Algorithm', 2, None, '___sec17'), ('Cancer Data again now with Decision Trees', 2, None, - '___sec17'), - ('Another example, the moons again', 2, None, '___sec18'), - ('Playing around with regions', 2, None, '___sec19'), - ('Regression trees', 2, None, '___sec20'), - ('Final regressor code', 2, None, '___sec21'), - ('Pros and cons of trees, pros', 2, None, '___sec22'), - ('Disadvantages', 2, None, '___sec23'), - ('Bagging', 2, None, '___sec24'), - ('Simple example, head or tail', 2, None, '___sec25'), - ('Random forests', 2, None, '___sec26'), - ('A simple scikit-learn example', 2, None, '___sec27'), - ('Please, not the moons again!', 2, None, '___sec28'), - ('Bagging examples', 2, None, '___sec29'), - ('Then random forests', 2, None, '___sec30'), - ('Boosting and more', 2, None, '___sec31')]} + '___sec18'), + ('Another example, the moons again', 2, None, '___sec19'), + ('Playing around with regions', 2, None, '___sec20'), + ('Regression trees', 2, None, '___sec21'), + ('Final regressor code', 2, None, '___sec22'), + ('Pros and cons of trees, pros', 2, None, '___sec23'), + ('Disadvantages', 2, None, '___sec24'), + ('Bagging', 2, None, '___sec25'), + ('Simple example, head or tail', 2, None, '___sec26'), + ('Random forests', 2, None, '___sec27'), + ('A simple scikit-learn example', 2, None, '___sec28'), + ('Please, not the moons again!', 2, None, '___sec29'), + ('Bagging examples', 2, None, '___sec30'), + ('Then random forests', 2, None, '___sec31'), + ('Boosting and more', 2, None, '___sec32')]} end of tocinfo --> @@ -139,23 +143,24 @@ MathJax.Hub.Config({
  • A Classification Tree
  • Growing a classification tree
  • Classification tree, how to split nodes
    • -
  • Entropy and the ID3 algorithm
    • -
  • Implementing the ID3 Algorithm
    • -
  • Cancer Data again now with Decision Trees
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Bagging
    • -
  • Simple example, head or tail
    • -
  • Random forests
    • -
  • A simple scikit-learn example
    • -
  • Please, not the moons again!
    • -
  • Bagging examples
    • -
  • Then random forests
    • -
  • Boosting and more
    • +
  • The CART (Classification and Regression Tree) algorithm
    • +
  • Entropy and the ID3 algorithm
    • +
  • Implementing the ID3 Algorithm
    • +
  • Cancer Data again now with Decision Trees
    • +
  • Another example, the moons again
    • +
  • Playing around with regions
    • +
  • Regression trees
    • +
  • Final regressor code
    • +
  • Pros and cons of trees, pros
    • +
  • Disadvantages
    • +
  • Bagging
    • +
  • Simple example, head or tail
    • +
  • Random forests
    • +
  • A simple scikit-learn example
    • +
  • Please, not the moons again!
    • +
  • Bagging examples
    • +
  • Then random forests
    • +
  • Boosting and more
    • @@ -171,74 +176,51 @@ MathJax.Hub.Config({ -

    Another example, the moons again

    +

    Cancer Data again now with Decision Trees

    -

    from __future__ import division, print_function, unicode_literals
-
-# Common imports
+
    import matplotlib.pyplot as plt
 import numpy as np
-import os
-
-# to make this notebook's output stable across runs
-np.random.seed(42)
-
-# To plot pretty figures
-import matplotlib
-import matplotlib.pyplot as plt
-from matplotlib.colors import ListedColormap
-plt.rcParams['axes.labelsize'] = 14
-plt.rcParams['xtick.labelsize'] = 12
-plt.rcParams['ytick.labelsize'] = 12
-
-
+from sklearn.model_selection import  train_test_split 
+from sklearn.datasets import load_breast_cancer
 from sklearn.svm import SVC
-from sklearn import datasets
+from sklearn.linear_model import LogisticRegression
 from sklearn.tree import DecisionTreeClassifier
-from sklearn.datasets import make_moons
-from sklearn.tree import export_graphviz
 
-Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)
+# Load the data
+cancer = load_breast_cancer()
 
-deep_tree_clf1 = DecisionTreeClassifier(random_state=42)
-deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)
-deep_tree_clf1.fit(Xm, ym)
-deep_tree_clf2.fit(Xm, ym)
-
-
-def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):
-    x1s = np.linspace(axes[0], axes[1], 100)
-    x2s = np.linspace(axes[2], axes[3], 100)
-    x1, x2 = np.meshgrid(x1s, x2s)
-    X_new = np.c_[x1.ravel(), x2.ravel()]
-    y_pred = clf.predict(X_new).reshape(x1.shape)
-    custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
-    plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
-    if not iris:
-        custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
-        plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
-    if plot_training:
-        plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa")
-        plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor")
-        plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica")
-        plt.axis(axes)
-    if iris:
-        plt.xlabel("Petal length", fontsize=14)
-        plt.ylabel("Petal width", fontsize=14)
-    else:
-        plt.xlabel(r"$x_1$", fontsize=18)
-        plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
-    if legend:
-        plt.legend(loc="lower right", fontsize=14)
-plt.figure(figsize=(11, 4))
-plt.subplot(121)
-plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
-plt.title("No restrictions", fontsize=16)
-plt.subplot(122)
-plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
-plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14)
-plt.show()
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+# Logistic Regression
+logreg = LogisticRegression(solver='lbfgs')
+logreg.fit(X_train, y_train)
+print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
+# Support vector machine
+svm = SVC(gamma='auto', C=100)
+svm.fit(X_train, y_train)
+print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
+# Decision Trees
+deep_tree_clf = DecisionTreeClassifier(max_depth=None)
+deep_tree_clf.fit(X_train, y_train)
+print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+# Logistic Regression
+logreg.fit(X_train_scaled, y_train)
+print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Support Vector Machine
+svm.fit(X_train_scaled, y_train)
+print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Decision Trees
+deep_tree_clf.fit(X_train_scaled, y_train)
+print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
 
    
 
    
 
    
@@ -266,7 +248,7 @@ plt.show()
   
  • 28
    • 
   
  • 29
    • 
   
  • ...
    • 
-  
  • 33
    • 
+  
  • 34
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs020.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs020.html
index 39d29c531..644725d7e 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs020.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs020.html
@@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source
               ('A Classification Tree', 2, None, '___sec12'),
               ('Growing a classification tree', 2, None, '___sec13'),
               ('Classification tree, how to split nodes', 2, None, '___sec14'),
-              ('Entropy and the ID3 algorithm', 2, None, '___sec15'),
-              ('Implementing the ID3 Algorithm', 2, None, '___sec16'),
+              ('The CART (Classification and Regression Tree) algorithm',
+               2,
+               None,
+               '___sec15'),
+              ('Entropy and the ID3 algorithm', 2, None, '___sec16'),
+              ('Implementing the ID3 Algorithm', 2, None, '___sec17'),
               ('Cancer Data again now with Decision Trees',
                2,
                None,
-               '___sec17'),
-              ('Another example, the moons again', 2, None, '___sec18'),
-              ('Playing around with regions', 2, None, '___sec19'),
-              ('Regression trees', 2, None, '___sec20'),
-              ('Final regressor code', 2, None, '___sec21'),
-              ('Pros and cons of trees, pros', 2, None, '___sec22'),
-              ('Disadvantages', 2, None, '___sec23'),
-              ('Bagging', 2, None, '___sec24'),
-              ('Simple example, head or tail', 2, None, '___sec25'),
-              ('Random forests', 2, None, '___sec26'),
-              ('A simple scikit-learn example', 2, None, '___sec27'),
-              ('Please, not the moons again!', 2, None, '___sec28'),
-              ('Bagging examples', 2, None, '___sec29'),
-              ('Then random forests', 2, None, '___sec30'),
-              ('Boosting and more', 2, None, '___sec31')]}
+               '___sec18'),
+              ('Another example, the moons again', 2, None, '___sec19'),
+              ('Playing around with regions', 2, None, '___sec20'),
+              ('Regression trees', 2, None, '___sec21'),
+              ('Final regressor code', 2, None, '___sec22'),
+              ('Pros and cons of trees, pros', 2, None, '___sec23'),
+              ('Disadvantages', 2, None, '___sec24'),
+              ('Bagging', 2, None, '___sec25'),
+              ('Simple example, head or tail', 2, None, '___sec26'),
+              ('Random forests', 2, None, '___sec27'),
+              ('A simple scikit-learn example', 2, None, '___sec28'),
+              ('Please, not the moons again!', 2, None, '___sec29'),
+              ('Bagging examples', 2, None, '___sec30'),
+              ('Then random forests', 2, None, '___sec31'),
+              ('Boosting and more', 2, None, '___sec32')]}
 end of tocinfo -->
 
 
@@ -139,23 +143,24 @@ MathJax.Hub.Config({
       
  • A Classification Tree
    • 
       
  • Growing a classification tree
    • 
       
  • Classification tree, how to split nodes
    • 
-      
  • Entropy and the ID3 algorithm
    • 
-      
  • Implementing the ID3 Algorithm
    • 
-      
  • Cancer Data again now with Decision Trees
    • 
-      
  • Another example, the moons again
    • 
-      
  • Playing around with regions
    • 
-      
  • Regression trees
    • 
-      
  • Final regressor code
    • 
-      
  • Pros and cons of trees, pros
    • 
-      
  • Disadvantages
    • 
-      
  • Bagging
    • 
-      
  • Simple example, head or tail
    • 
-      
  • Random forests
    • 
-      
  • A simple scikit-learn example
    • 
-      
  • Please, not the moons again!
    • 
-      
  • Bagging examples
    • 
-      
  • Then random forests
    • 
-      
  • Boosting and more
    • 
+      
  • The CART (Classification and Regression Tree) algorithm
    • 
+      
  • Entropy and the ID3 algorithm
    • 
+      
  • Implementing the ID3 Algorithm
    • 
+      
  • Cancer Data again now with Decision Trees
    • 
+      
  • Another example, the moons again
    • 
+      
  • Playing around with regions
    • 
+      
  • Regression trees
    • 
+      
  • Final regressor code
    • 
+      
  • Pros and cons of trees, pros
    • 
+      
  • Disadvantages
    • 
+      
  • Bagging
    • 
+      
  • Simple example, head or tail
    • 
+      
  • Random forests
    • 
+      
  • A simple scikit-learn example
    • 
+      
  • Please, not the moons again!
    • 
+      
  • Bagging examples
    • 
+      
  • Then random forests
    • 
+      
  • Boosting and more
    • 
 
         
       
@@ -171,29 +176,73 @@ MathJax.Hub.Config({
 
 
 
-
    Playing around with regions 
    
+
    Another example, the moons again 
    
 
    
 
 
-
    np.random.seed(6)
-Xs = np.random.rand(100, 2) - 0.5
-ys = (Xs[:, 0] > 0).astype(np.float32) * 2
+
    from __future__ import division, print_function, unicode_literals
 
-angle = np.pi / 4
-rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
-Xsr = Xs.dot(rotation_matrix)
+# Common imports
+import numpy as np
+import os
 
-tree_clf_s = DecisionTreeClassifier(random_state=42)
-tree_clf_s.fit(Xs, ys)
-tree_clf_sr = DecisionTreeClassifier(random_state=42)
-tree_clf_sr.fit(Xsr, ys)
+# to make this notebook's output stable across runs
+np.random.seed(42)
 
+# To plot pretty figures
+import matplotlib
+import matplotlib.pyplot as plt
+from matplotlib.colors import ListedColormap
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+from sklearn.svm import SVC
+from sklearn import datasets
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.datasets import make_moons
+from sklearn.tree import export_graphviz
+
+Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)
+
+deep_tree_clf1 = DecisionTreeClassifier(random_state=42)
+deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)
+deep_tree_clf1.fit(Xm, ym)
+deep_tree_clf2.fit(Xm, ym)
+
+
+def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):
+    x1s = np.linspace(axes[0], axes[1], 100)
+    x2s = np.linspace(axes[2], axes[3], 100)
+    x1, x2 = np.meshgrid(x1s, x2s)
+    X_new = np.c_[x1.ravel(), x2.ravel()]
+    y_pred = clf.predict(X_new).reshape(x1.shape)
+    custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
+    plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
+    if not iris:
+        custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
+        plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
+    if plot_training:
+        plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa")
+        plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor")
+        plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica")
+        plt.axis(axes)
+    if iris:
+        plt.xlabel("Petal length", fontsize=14)
+        plt.ylabel("Petal width", fontsize=14)
+    else:
+        plt.xlabel(r"$x_1$", fontsize=18)
+        plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
+    if legend:
+        plt.legend(loc="lower right", fontsize=14)
 plt.figure(figsize=(11, 4))
 plt.subplot(121)
-plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
+plt.title("No restrictions", fontsize=16)
 plt.subplot(122)
-plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
-
+plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
+plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14)
 plt.show()
 
    
 
    
@@ -222,7 +271,7 @@ plt.show()
   
  • 29
    • 
   
  • 30
    • 
   
  • ...
    • 
-  
  • 33
    • 
+  
  • 34
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs021.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs021.html
index 077ed8885..03d9652fb 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs021.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs021.html
@@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source
               ('A Classification Tree', 2, None, '___sec12'),
               ('Growing a classification tree', 2, None, '___sec13'),
               ('Classification tree, how to split nodes', 2, None, '___sec14'),
-              ('Entropy and the ID3 algorithm', 2, None, '___sec15'),
-              ('Implementing the ID3 Algorithm', 2, None, '___sec16'),
+              ('The CART (Classification and Regression Tree) algorithm',
+               2,
+               None,
+               '___sec15'),
+              ('Entropy and the ID3 algorithm', 2, None, '___sec16'),
+              ('Implementing the ID3 Algorithm', 2, None, '___sec17'),
               ('Cancer Data again now with Decision Trees',
                2,
                None,
-               '___sec17'),
-              ('Another example, the moons again', 2, None, '___sec18'),
-              ('Playing around with regions', 2, None, '___sec19'),
-              ('Regression trees', 2, None, '___sec20'),
-              ('Final regressor code', 2, None, '___sec21'),
-              ('Pros and cons of trees, pros', 2, None, '___sec22'),
-              ('Disadvantages', 2, None, '___sec23'),
-              ('Bagging', 2, None, '___sec24'),
-              ('Simple example, head or tail', 2, None, '___sec25'),
-              ('Random forests', 2, None, '___sec26'),
-              ('A simple scikit-learn example', 2, None, '___sec27'),
-              ('Please, not the moons again!', 2, None, '___sec28'),
-              ('Bagging examples', 2, None, '___sec29'),
-              ('Then random forests', 2, None, '___sec30'),
-              ('Boosting and more', 2, None, '___sec31')]}
+               '___sec18'),
+              ('Another example, the moons again', 2, None, '___sec19'),
+              ('Playing around with regions', 2, None, '___sec20'),
+              ('Regression trees', 2, None, '___sec21'),
+              ('Final regressor code', 2, None, '___sec22'),
+              ('Pros and cons of trees, pros', 2, None, '___sec23'),
+              ('Disadvantages', 2, None, '___sec24'),
+              ('Bagging', 2, None, '___sec25'),
+              ('Simple example, head or tail', 2, None, '___sec26'),
+              ('Random forests', 2, None, '___sec27'),
+              ('A simple scikit-learn example', 2, None, '___sec28'),
+              ('Please, not the moons again!', 2, None, '___sec29'),
+              ('Bagging examples', 2, None, '___sec30'),
+              ('Then random forests', 2, None, '___sec31'),
+              ('Boosting and more', 2, None, '___sec32')]}
 end of tocinfo -->
 
 
@@ -139,23 +143,24 @@ MathJax.Hub.Config({
       
  • A Classification Tree
    • 
       
  • Growing a classification tree
    • 
       
  • Classification tree, how to split nodes
    • 
-      
  • Entropy and the ID3 algorithm
    • 
-      
  • Implementing the ID3 Algorithm
    • 
-      
  • Cancer Data again now with Decision Trees
    • 
-      
  • Another example, the moons again
    • 
-      
  • Playing around with regions
    • 
-      
  • Regression trees
    • 
-      
  • Final regressor code
    • 
-      
  • Pros and cons of trees, pros
    • 
-      
  • Disadvantages
    • 
-      
  • Bagging
    • 
-      
  • Simple example, head or tail
    • 
-      
  • Random forests
    • 
-      
  • A simple scikit-learn example
    • 
-      
  • Please, not the moons again!
    • 
-      
  • Bagging examples
    • 
-      
  • Then random forests
    • 
-      
  • Boosting and more
    • 
+      
  • The CART (Classification and Regression Tree) algorithm
    • 
+      
  • Entropy and the ID3 algorithm
    • 
+      
  • Implementing the ID3 Algorithm
    • 
+      
  • Cancer Data again now with Decision Trees
    • 
+      
  • Another example, the moons again
    • 
+      
  • Playing around with regions
    • 
+      
  • Regression trees
    • 
+      
  • Final regressor code
    • 
+      
  • Pros and cons of trees, pros
    • 
+      
  • Disadvantages
    • 
+      
  • Bagging
    • 
+      
  • Simple example, head or tail
    • 
+      
  • Random forests
    • 
+      
  • A simple scikit-learn example
    • 
+      
  • Please, not the moons again!
    • 
+      
  • Bagging examples
    • 
+      
  • Then random forests
    • 
+      
  • Boosting and more
    • 
 
         
       
@@ -171,24 +176,30 @@ MathJax.Hub.Config({
 
 
 
-
    Regression trees 
    
+
    Playing around with regions 
    
 
    
 
 
-
    # Quadratic training set + noise
-np.random.seed(42)
-m = 200
-X = np.random.rand(m, 1)
-y = 4 * (X - 0.5) ** 2
-y = y + np.random.randn(m, 1) / 10
-
    
-
    
+
    np.random.seed(6)
+Xs = np.random.rand(100, 2) - 0.5
+ys = (Xs[:, 0] > 0).astype(np.float32) * 2
 
-
-
    from sklearn.tree import DecisionTreeRegressor
+angle = np.pi / 4
+rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
+Xsr = Xs.dot(rotation_matrix)
 
-tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg.fit(X, y)
+tree_clf_s = DecisionTreeClassifier(random_state=42)
+tree_clf_s.fit(Xs, ys)
+tree_clf_sr = DecisionTreeClassifier(random_state=42)
+tree_clf_sr.fit(Xsr, ys)
+
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+plt.subplot(122)
+plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+
+plt.show()
 
    
 
    
 
    
@@ -216,7 +227,7 @@ tree_reg.fit(X, y)
   
  • 30
    • 
   
  • 31
    • 
   
  • ...
    • 
-  
  • 33
    • 
+  
  • 34
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs022.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs022.html
index d692c2d6e..cd54b9b78 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs022.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs022.html
@@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source
               ('A Classification Tree', 2, None, '___sec12'),
               ('Growing a classification tree', 2, None, '___sec13'),
               ('Classification tree, how to split nodes', 2, None, '___sec14'),
-              ('Entropy and the ID3 algorithm', 2, None, '___sec15'),
-              ('Implementing the ID3 Algorithm', 2, None, '___sec16'),
+              ('The CART (Classification and Regression Tree) algorithm',
+               2,
+               None,
+               '___sec15'),
+              ('Entropy and the ID3 algorithm', 2, None, '___sec16'),
+              ('Implementing the ID3 Algorithm', 2, None, '___sec17'),
               ('Cancer Data again now with Decision Trees',
                2,
                None,
-               '___sec17'),
-              ('Another example, the moons again', 2, None, '___sec18'),
-              ('Playing around with regions', 2, None, '___sec19'),
-              ('Regression trees', 2, None, '___sec20'),
-              ('Final regressor code', 2, None, '___sec21'),
-              ('Pros and cons of trees, pros', 2, None, '___sec22'),
-              ('Disadvantages', 2, None, '___sec23'),
-              ('Bagging', 2, None, '___sec24'),
-              ('Simple example, head or tail', 2, None, '___sec25'),
-              ('Random forests', 2, None, '___sec26'),
-              ('A simple scikit-learn example', 2, None, '___sec27'),
-              ('Please, not the moons again!', 2, None, '___sec28'),
-              ('Bagging examples', 2, None, '___sec29'),
-              ('Then random forests', 2, None, '___sec30'),
-              ('Boosting and more', 2, None, '___sec31')]}
+               '___sec18'),
+              ('Another example, the moons again', 2, None, '___sec19'),
+              ('Playing around with regions', 2, None, '___sec20'),
+              ('Regression trees', 2, None, '___sec21'),
+              ('Final regressor code', 2, None, '___sec22'),
+              ('Pros and cons of trees, pros', 2, None, '___sec23'),
+              ('Disadvantages', 2, None, '___sec24'),
+              ('Bagging', 2, None, '___sec25'),
+              ('Simple example, head or tail', 2, None, '___sec26'),
+              ('Random forests', 2, None, '___sec27'),
+              ('A simple scikit-learn example', 2, None, '___sec28'),
+              ('Please, not the moons again!', 2, None, '___sec29'),
+              ('Bagging examples', 2, None, '___sec30'),
+              ('Then random forests', 2, None, '___sec31'),
+              ('Boosting and more', 2, None, '___sec32')]}
 end of tocinfo -->
 
 
@@ -139,23 +143,24 @@ MathJax.Hub.Config({
       
  • A Classification Tree
    • 
       
  • Growing a classification tree
    • 
       
  • Classification tree, how to split nodes
    • 
-      
  • Entropy and the ID3 algorithm
    • 
-      
  • Implementing the ID3 Algorithm
    • 
-      
  • Cancer Data again now with Decision Trees
    • 
-      
  • Another example, the moons again
    • 
-      
  • Playing around with regions
    • 
-      
  • Regression trees
    • 
-      
  • Final regressor code
    • 
-      
  • Pros and cons of trees, pros
    • 
-      
  • Disadvantages
    • 
-      
  • Bagging
    • 
-      
  • Simple example, head or tail
    • 
-      
  • Random forests
    • 
-      
  • A simple scikit-learn example
    • 
-      
  • Please, not the moons again!
    • 
-      
  • Bagging examples
    • 
-      
  • Then random forests
    • 
-      
  • Boosting and more
    • 
+      
  • The CART (Classification and Regression Tree) algorithm
    • 
+      
  • Entropy and the ID3 algorithm
    • 
+      
  • Implementing the ID3 Algorithm
    • 
+      
  • Cancer Data again now with Decision Trees
    • 
+      
  • Another example, the moons again
    • 
+      
  • Playing around with regions
    • 
+      
  • Regression trees
    • 
+      
  • Final regressor code
    • 
+      
  • Pros and cons of trees, pros
    • 
+      
  • Disadvantages
    • 
+      
  • Bagging
    • 
+      
  • Simple example, head or tail
    • 
+      
  • Random forests
    • 
+      
  • A simple scikit-learn example
    • 
+      
  • Please, not the moons again!
    • 
+      
  • Bagging examples
    • 
+      
  • Then random forests
    • 
+      
  • Boosting and more
    • 
 
         
       
@@ -171,80 +176,24 @@ MathJax.Hub.Config({
 
 
 
-
    Final regressor code 
    
+
    Regression trees 
    
+
    
+
+
+
    # Quadratic training set + noise
+np.random.seed(42)
+m = 200
+X = np.random.rand(m, 1)
+y = 4 * (X - 0.5) ** 2
+y = y + np.random.randn(m, 1) / 10
+
    
 
    
 
 
 
    from sklearn.tree import DecisionTreeRegressor
 
-tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)
-tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)
-tree_reg1.fit(X, y)
-tree_reg2.fit(X, y)
-
-def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel="$y$"):
-    x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)
-    y_pred = tree_reg.predict(x1)
-    plt.axis(axes)
-    plt.xlabel("$x_1$", fontsize=18)
-    if ylabel:
-        plt.ylabel(ylabel, fontsize=18, rotation=0)
-    plt.plot(X, y, "b.")
-    plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$")
-
-plt.figure(figsize=(11, 4))
-plt.subplot(121)
-plot_regression_predictions(tree_reg1, X, y)
-for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
-    plt.plot([split, split], [-0.2, 1], style, linewidth=2)
-plt.text(0.21, 0.65, "Depth=0", fontsize=15)
-plt.text(0.01, 0.2, "Depth=1", fontsize=13)
-plt.text(0.65, 0.8, "Depth=1", fontsize=13)
-plt.legend(loc="upper center", fontsize=18)
-plt.title("max_depth=2", fontsize=14)
-
-plt.subplot(122)
-plot_regression_predictions(tree_reg2, X, y, ylabel=None)
-for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
-    plt.plot([split, split], [-0.2, 1], style, linewidth=2)
-for split in (0.0458, 0.1298, 0.2873, 0.9040):
-    plt.plot([split, split], [-0.2, 1], "k:", linewidth=1)
-plt.text(0.3, 0.5, "Depth=2", fontsize=13)
-plt.title("max_depth=3", fontsize=14)
-
-plt.show()
-
    
-
    
-
-
-
    tree_reg1 = DecisionTreeRegressor(random_state=42)
-tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)
-tree_reg1.fit(X, y)
-tree_reg2.fit(X, y)
-
-x1 = np.linspace(0, 1, 500).reshape(-1, 1)
-y_pred1 = tree_reg1.predict(x1)
-y_pred2 = tree_reg2.predict(x1)
-
-plt.figure(figsize=(11, 4))
-
-plt.subplot(121)
-plt.plot(X, y, "b.")
-plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$")
-plt.axis([0, 1, -0.2, 1.1])
-plt.xlabel("$x_1$", fontsize=18)
-plt.ylabel("$y$", fontsize=18, rotation=0)
-plt.legend(loc="upper center", fontsize=18)
-plt.title("No restrictions", fontsize=14)
-
-plt.subplot(122)
-plt.plot(X, y, "b.")
-plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$")
-plt.axis([0, 1, -0.2, 1.1])
-plt.xlabel("$x_1$", fontsize=18)
-plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14)
-
-plt.show()
+tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg.fit(X, y)
 
    
 
    
 
    
@@ -272,7 +221,7 @@ plt.show()
   
  • 31
    • 
   
  • 32
    • 
   
  • ...
    • 
-  
  • 33
    • 
+  
  • 34
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs023.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs023.html
index fdd169e96..67de6c38d 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs023.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs023.html
@@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source
               ('A Classification Tree', 2, None, '___sec12'),
               ('Growing a classification tree', 2, None, '___sec13'),
               ('Classification tree, how to split nodes', 2, None, '___sec14'),
-              ('Entropy and the ID3 algorithm', 2, None, '___sec15'),
-              ('Implementing the ID3 Algorithm', 2, None, '___sec16'),
+              ('The CART (Classification and Regression Tree) algorithm',
+               2,
+               None,
+               '___sec15'),
+              ('Entropy and the ID3 algorithm', 2, None, '___sec16'),
+              ('Implementing the ID3 Algorithm', 2, None, '___sec17'),
               ('Cancer Data again now with Decision Trees',
                2,
                None,
-               '___sec17'),
-              ('Another example, the moons again', 2, None, '___sec18'),
-              ('Playing around with regions', 2, None, '___sec19'),
-              ('Regression trees', 2, None, '___sec20'),
-              ('Final regressor code', 2, None, '___sec21'),
-              ('Pros and cons of trees, pros', 2, None, '___sec22'),
-              ('Disadvantages', 2, None, '___sec23'),
-              ('Bagging', 2, None, '___sec24'),
-              ('Simple example, head or tail', 2, None, '___sec25'),
-              ('Random forests', 2, None, '___sec26'),
-              ('A simple scikit-learn example', 2, None, '___sec27'),
-              ('Please, not the moons again!', 2, None, '___sec28'),
-              ('Bagging examples', 2, None, '___sec29'),
-              ('Then random forests', 2, None, '___sec30'),
-              ('Boosting and more', 2, None, '___sec31')]}
+               '___sec18'),
+              ('Another example, the moons again', 2, None, '___sec19'),
+              ('Playing around with regions', 2, None, '___sec20'),
+              ('Regression trees', 2, None, '___sec21'),
+              ('Final regressor code', 2, None, '___sec22'),
+              ('Pros and cons of trees, pros', 2, None, '___sec23'),
+              ('Disadvantages', 2, None, '___sec24'),
+              ('Bagging', 2, None, '___sec25'),
+              ('Simple example, head or tail', 2, None, '___sec26'),
+              ('Random forests', 2, None, '___sec27'),
+              ('A simple scikit-learn example', 2, None, '___sec28'),
+              ('Please, not the moons again!', 2, None, '___sec29'),
+              ('Bagging examples', 2, None, '___sec30'),
+              ('Then random forests', 2, None, '___sec31'),
+              ('Boosting and more', 2, None, '___sec32')]}
 end of tocinfo -->
 
 
@@ -139,23 +143,24 @@ MathJax.Hub.Config({
       
  • A Classification Tree
    • 
       
  • Growing a classification tree
    • 
       
  • Classification tree, how to split nodes
    • 
-      
  • Entropy and the ID3 algorithm
    • 
-      
  • Implementing the ID3 Algorithm
    • 
-      
  • Cancer Data again now with Decision Trees
    • 
-      
  • Another example, the moons again
    • 
-      
  • Playing around with regions
    • 
-      
  • Regression trees
    • 
-      
  • Final regressor code
    • 
-      
  • Pros and cons of trees, pros
    • 
-      
  • Disadvantages
    • 
-      
  • Bagging
    • 
-      
  • Simple example, head or tail
    • 
-      
  • Random forests
    • 
-      
  • A simple scikit-learn example
    • 
-      
  • Please, not the moons again!
    • 
-      
  • Bagging examples
    • 
-      
  • Then random forests
    • 
-      
  • Boosting and more
    • 
+      
  • The CART (Classification and Regression Tree) algorithm
    • 
+      
  • Entropy and the ID3 algorithm
    • 
+      
  • Implementing the ID3 Algorithm
    • 
+      
  • Cancer Data again now with Decision Trees
    • 
+      
  • Another example, the moons again
    • 
+      
  • Playing around with regions
    • 
+      
  • Regression trees
    • 
+      
  • Final regressor code
    • 
+      
  • Pros and cons of trees, pros
    • 
+      
  • Disadvantages
    • 
+      
  • Bagging
    • 
+      
  • Simple example, head or tail
    • 
+      
  • Random forests
    • 
+      
  • A simple scikit-learn example
    • 
+      
  • Please, not the moons again!
    • 
+      
  • Bagging examples
    • 
+      
  • Then random forests
    • 
+      
  • Boosting and more
    • 
 
         
       
@@ -171,18 +176,82 @@ MathJax.Hub.Config({
 
 
 
-
    Pros and cons of trees, pros 
    
+
    Final regressor code 
    
+
    
 
-
      
-
    •  White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines)
      • 
-
    •  Trees are very easy to explain to people. In fact, they are even easier to explain than linear regression!
      • 
-
    •  No feature normalization needed
      • 
-
    •  Tree models can handle both continuous and categorical data (Classification and Regression Trees)
      • 
-
    •  Can model nonlinear relationships
      • 
-
    •  Can model interactions between the different descriptive features
      • 
-
    •  Trees can be displayed graphically, and are easily interpreted even by a non-expert (especially if they are small)
      • 
-
    
+
+
    from sklearn.tree import DecisionTreeRegressor
 
+tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)
+tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)
+tree_reg1.fit(X, y)
+tree_reg2.fit(X, y)
+
+def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel="$y$"):
+    x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)
+    y_pred = tree_reg.predict(x1)
+    plt.axis(axes)
+    plt.xlabel("$x_1$", fontsize=18)
+    if ylabel:
+        plt.ylabel(ylabel, fontsize=18, rotation=0)
+    plt.plot(X, y, "b.")
+    plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$")
+
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_regression_predictions(tree_reg1, X, y)
+for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
+    plt.plot([split, split], [-0.2, 1], style, linewidth=2)
+plt.text(0.21, 0.65, "Depth=0", fontsize=15)
+plt.text(0.01, 0.2, "Depth=1", fontsize=13)
+plt.text(0.65, 0.8, "Depth=1", fontsize=13)
+plt.legend(loc="upper center", fontsize=18)
+plt.title("max_depth=2", fontsize=14)
+
+plt.subplot(122)
+plot_regression_predictions(tree_reg2, X, y, ylabel=None)
+for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
+    plt.plot([split, split], [-0.2, 1], style, linewidth=2)
+for split in (0.0458, 0.1298, 0.2873, 0.9040):
+    plt.plot([split, split], [-0.2, 1], "k:", linewidth=1)
+plt.text(0.3, 0.5, "Depth=2", fontsize=13)
+plt.title("max_depth=3", fontsize=14)
+
+plt.show()
+
    
+
    
+
+
+
    tree_reg1 = DecisionTreeRegressor(random_state=42)
+tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)
+tree_reg1.fit(X, y)
+tree_reg2.fit(X, y)
+
+x1 = np.linspace(0, 1, 500).reshape(-1, 1)
+y_pred1 = tree_reg1.predict(x1)
+y_pred2 = tree_reg2.predict(x1)
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.plot(X, y, "b.")
+plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$")
+plt.axis([0, 1, -0.2, 1.1])
+plt.xlabel("$x_1$", fontsize=18)
+plt.ylabel("$y$", fontsize=18, rotation=0)
+plt.legend(loc="upper center", fontsize=18)
+plt.title("No restrictions", fontsize=14)
+
+plt.subplot(122)
+plt.plot(X, y, "b.")
+plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$")
+plt.axis([0, 1, -0.2, 1.1])
+plt.xlabel("$x_1$", fontsize=18)
+plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14)
+
+plt.show()
+
    
+
    
 
    
 
 
      
@@ -207,6 +276,8 @@ MathJax.Hub.Config({
   
    • 31
      • 
   
    • 32
      • 
   
    • 33
      • 
+  
    • ...
      • 
+  
    • 34
      • 
   
    • »
      • 
 
    
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs024.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs024.html
index 86deef56b..245bac28b 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs024.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs024.html
@@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source
               ('A Classification Tree', 2, None, '___sec12'),
               ('Growing a classification tree', 2, None, '___sec13'),
               ('Classification tree, how to split nodes', 2, None, '___sec14'),
-              ('Entropy and the ID3 algorithm', 2, None, '___sec15'),
-              ('Implementing the ID3 Algorithm', 2, None, '___sec16'),
+              ('The CART (Classification and Regression Tree) algorithm',
+               2,
+               None,
+               '___sec15'),
+              ('Entropy and the ID3 algorithm', 2, None, '___sec16'),
+              ('Implementing the ID3 Algorithm', 2, None, '___sec17'),
               ('Cancer Data again now with Decision Trees',
                2,
                None,
-               '___sec17'),
-              ('Another example, the moons again', 2, None, '___sec18'),
-              ('Playing around with regions', 2, None, '___sec19'),
-              ('Regression trees', 2, None, '___sec20'),
-              ('Final regressor code', 2, None, '___sec21'),
-              ('Pros and cons of trees, pros', 2, None, '___sec22'),
-              ('Disadvantages', 2, None, '___sec23'),
-              ('Bagging', 2, None, '___sec24'),
-              ('Simple example, head or tail', 2, None, '___sec25'),
-              ('Random forests', 2, None, '___sec26'),
-              ('A simple scikit-learn example', 2, None, '___sec27'),
-              ('Please, not the moons again!', 2, None, '___sec28'),
-              ('Bagging examples', 2, None, '___sec29'),
-              ('Then random forests', 2, None, '___sec30'),
-              ('Boosting and more', 2, None, '___sec31')]}
+               '___sec18'),
+              ('Another example, the moons again', 2, None, '___sec19'),
+              ('Playing around with regions', 2, None, '___sec20'),
+              ('Regression trees', 2, None, '___sec21'),
+              ('Final regressor code', 2, None, '___sec22'),
+              ('Pros and cons of trees, pros', 2, None, '___sec23'),
+              ('Disadvantages', 2, None, '___sec24'),
+              ('Bagging', 2, None, '___sec25'),
+              ('Simple example, head or tail', 2, None, '___sec26'),
+              ('Random forests', 2, None, '___sec27'),
+              ('A simple scikit-learn example', 2, None, '___sec28'),
+              ('Please, not the moons again!', 2, None, '___sec29'),
+              ('Bagging examples', 2, None, '___sec30'),
+              ('Then random forests', 2, None, '___sec31'),
+              ('Boosting and more', 2, None, '___sec32')]}
 end of tocinfo -->
 
 
@@ -139,23 +143,24 @@ MathJax.Hub.Config({
       
  • A Classification Tree
    • 
       
  • Growing a classification tree
    • 
       
  • Classification tree, how to split nodes
    • 
-      
  • Entropy and the ID3 algorithm
    • 
-      
  • Implementing the ID3 Algorithm
    • 
-      
  • Cancer Data again now with Decision Trees
    • 
-      
  • Another example, the moons again
    • 
-      
  • Playing around with regions
    • 
-      
  • Regression trees
    • 
-      
  • Final regressor code
    • 
-      
  • Pros and cons of trees, pros
    • 
-      
  • Disadvantages
    • 
-      
  • Bagging
    • 
-      
  • Simple example, head or tail
    • 
-      
  • Random forests
    • 
-      
  • A simple scikit-learn example
    • 
-      
  • Please, not the moons again!
    • 
-      
  • Bagging examples
    • 
-      
  • Then random forests
    • 
-      
  • Boosting and more
    • 
+      
  • The CART (Classification and Regression Tree) algorithm
    • 
+      
  • Entropy and the ID3 algorithm
    • 
+      
  • Implementing the ID3 Algorithm
    • 
+      
  • Cancer Data again now with Decision Trees
    • 
+      
  • Another example, the moons again
    • 
+      
  • Playing around with regions
    • 
+      
  • Regression trees
    • 
+      
  • Final regressor code
    • 
+      
  • Pros and cons of trees, pros
    • 
+      
  • Disadvantages
    • 
+      
  • Bagging
    • 
+      
  • Simple example, head or tail
    • 
+      
  • Random forests
    • 
+      
  • A simple scikit-learn example
    • 
+      
  • Please, not the moons again!
    • 
+      
  • Bagging examples
    • 
+      
  • Then random forests
    • 
+      
  • Boosting and more
    • 
 
         
       
@@ -171,21 +176,18 @@ MathJax.Hub.Config({
 
 
 
-
    Disadvantages 
    
+
    Pros and cons of trees, pros 
    
 
 
      
-
    •  Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches
      • 
-
    •  If continuous features are used the tree may become quite large and hence less interpretable
      • 
-
    •  Decision trees are prone to overfit the training data and hence do not well generalize the data if no stopping criteria or improvements like pruning, boosting or bagging are implemented
      • 
-
    •  Small changes in the data may lead to a completely different tree. This issue can be addressed by using ensemble methods like bagging, boosting or random forests
      • 
-
    •  Unbalanced datasets where some target feature values occur much more frequently than others may lead to biased trees since the frequently occurring feature values are preferred over the less frequently occurring ones.
      •  
-
    •  If the number of features is relatively large (high dimensional) and the number of instances is relatively low, the tree might overfit the data
      • 
-
    •  Features with many levels may be preferred over features with less levels since for them it is more easy to split the dataset such that the sub datasets only contain pure target feature values. This issue can be addressed by preferring for instance the information gain ratio as splitting criteria over information gain
      • 
+
    •  White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines)
      • 
+
    •  Trees are very easy to explain to people. In fact, they are even easier to explain than linear regression!
      • 
+
    •  No feature normalization needed
      • 
+
    •  Tree models can handle both continuous and categorical data (Classification and Regression Trees)
      • 
+
    •  Can model nonlinear relationships
      • 
+
    •  Can model interactions between the different descriptive features
      • 
+
    •  Trees can be displayed graphically, and are easily interpreted even by a non-expert (especially if they are small)
      • 
 
    
 
-However, by aggregating many decision trees, using methods like bagging, random forests, and boosting, the predictive performance of trees can be substantially improved.
-
-
    
 
    
 
 
      
@@ -209,6 +211,7 @@ However, by aggregating many decision trees, using methods like bagging, random
   
    • 31
      • 
   
    • 32
      • 
   
    • 33
      • 
+  
    • 34
      • 
   
    • »
      • 
 
    
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs025.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs025.html
index 497d3fe62..199639975 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs025.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs025.html
@@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source
               ('A Classification Tree', 2, None, '___sec12'),
               ('Growing a classification tree', 2, None, '___sec13'),
               ('Classification tree, how to split nodes', 2, None, '___sec14'),
-              ('Entropy and the ID3 algorithm', 2, None, '___sec15'),
-              ('Implementing the ID3 Algorithm', 2, None, '___sec16'),
+              ('The CART (Classification and Regression Tree) algorithm',
+               2,
+               None,
+               '___sec15'),
+              ('Entropy and the ID3 algorithm', 2, None, '___sec16'),
+              ('Implementing the ID3 Algorithm', 2, None, '___sec17'),
               ('Cancer Data again now with Decision Trees',
                2,
                None,
-               '___sec17'),
-              ('Another example, the moons again', 2, None, '___sec18'),
-              ('Playing around with regions', 2, None, '___sec19'),
-              ('Regression trees', 2, None, '___sec20'),
-              ('Final regressor code', 2, None, '___sec21'),
-              ('Pros and cons of trees, pros', 2, None, '___sec22'),
-              ('Disadvantages', 2, None, '___sec23'),
-              ('Bagging', 2, None, '___sec24'),
-              ('Simple example, head or tail', 2, None, '___sec25'),
-              ('Random forests', 2, None, '___sec26'),
-              ('A simple scikit-learn example', 2, None, '___sec27'),
-              ('Please, not the moons again!', 2, None, '___sec28'),
-              ('Bagging examples', 2, None, '___sec29'),
-              ('Then random forests', 2, None, '___sec30'),
-              ('Boosting and more', 2, None, '___sec31')]}
+               '___sec18'),
+              ('Another example, the moons again', 2, None, '___sec19'),
+              ('Playing around with regions', 2, None, '___sec20'),
+              ('Regression trees', 2, None, '___sec21'),
+              ('Final regressor code', 2, None, '___sec22'),
+              ('Pros and cons of trees, pros', 2, None, '___sec23'),
+              ('Disadvantages', 2, None, '___sec24'),
+              ('Bagging', 2, None, '___sec25'),
+              ('Simple example, head or tail', 2, None, '___sec26'),
+              ('Random forests', 2, None, '___sec27'),
+              ('A simple scikit-learn example', 2, None, '___sec28'),
+              ('Please, not the moons again!', 2, None, '___sec29'),
+              ('Bagging examples', 2, None, '___sec30'),
+              ('Then random forests', 2, None, '___sec31'),
+              ('Boosting and more', 2, None, '___sec32')]}
 end of tocinfo -->
 
 
@@ -139,23 +143,24 @@ MathJax.Hub.Config({
       
  • A Classification Tree
    • 
       
  • Growing a classification tree
    • 
       
  • Classification tree, how to split nodes
    • 
-      
  • Entropy and the ID3 algorithm
    • 
-      
  • Implementing the ID3 Algorithm
    • 
-      
  • Cancer Data again now with Decision Trees
    • 
-      
  • Another example, the moons again
    • 
-      
  • Playing around with regions
    • 
-      
  • Regression trees
    • 
-      
  • Final regressor code
    • 
-      
  • Pros and cons of trees, pros
    • 
-      
  • Disadvantages
    • 
-      
  • Bagging
    • 
-      
  • Simple example, head or tail
    • 
-      
  • Random forests
    • 
-      
  • A simple scikit-learn example
    • 
-      
  • Please, not the moons again!
    • 
-      
  • Bagging examples
    • 
-      
  • Then random forests
    • 
-      
  • Boosting and more
    • 
+      
  • The CART (Classification and Regression Tree) algorithm
    • 
+      
  • Entropy and the ID3 algorithm
    • 
+      
  • Implementing the ID3 Algorithm
    • 
+      
  • Cancer Data again now with Decision Trees
    • 
+      
  • Another example, the moons again
    • 
+      
  • Playing around with regions
    • 
+      
  • Regression trees
    • 
+      
  • Final regressor code
    • 
+      
  • Pros and cons of trees, pros
    • 
+      
  • Disadvantages
    • 
+      
  • Bagging
    • 
+      
  • Simple example, head or tail
    • 
+      
  • Random forests
    • 
+      
  • A simple scikit-learn example
    • 
+      
  • Please, not the moons again!
    • 
+      
  • Bagging examples
    • 
+      
  • Then random forests
    • 
+      
  • Boosting and more
    • 
 
         
       
@@ -171,45 +176,19 @@ MathJax.Hub.Config({
 
 
 
-
    Bagging 
    
+
    Disadvantages 
    
 
-
    
-The plain decision trees suffer from high
-variance. This means that if we split the training data into two parts
-at random, and fit a decision tree to both halves, the results that we
-get could be quite different. In contrast, a procedure with low
-variance will yield similar results if applied repeatedly to distinct
-data sets; linear regression tends to have low variance, if the ratio
-of \( n \) to \( p \) is moderately large.
+
      
+
    •  Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches
      • 
+
    •  If continuous features are used the tree may become quite large and hence less interpretable
      • 
+
    •  Decision trees are prone to overfit the training data and hence do not well generalize the data if no stopping criteria or improvements like pruning, boosting or bagging are implemented
      • 
+
    •  Small changes in the data may lead to a completely different tree. This issue can be addressed by using ensemble methods like bagging, boosting or random forests
      • 
+
    •  Unbalanced datasets where some target feature values occur much more frequently than others may lead to biased trees since the frequently occurring feature values are preferred over the less frequently occurring ones.
      •  
+
    •  If the number of features is relatively large (high dimensional) and the number of instances is relatively low, the tree might overfit the data
      • 
+
    •  Features with many levels may be preferred over features with less levels since for them it is more easy to split the dataset such that the sub datasets only contain pure target feature values. This issue can be addressed by preferring for instance the information gain ratio as splitting criteria over information gain
      • 
+
    
 
-
    
-Bootstrap aggregation, or just bagging, is a
-general-purpose procedure for reducing the variance of a statistical
-learning method.
-
-
    
-Bagging typically results in improved accuracy
-over prediction using a single tree. Unfortunately, however, it can be
-difficult to interpret the resulting model. Recall that one of the
-advantages of decision trees is the attractive and easily interpreted
-diagram that results.
-
-
    
-However, when we bag a large number of trees, it is no longer
-possible to represent the resulting statistical learning procedure
-using a single tree, and it is no longer clear which variables are
-most important to the procedure. Thus, bagging improves prediction
-accuracy at the expense of interpretability.  Although the collection
-of bagged trees is much more difficult to interpret than a single
-tree, one can obtain an overall summary of the importance of each
-predictor using the MSE (for bagging regression trees) or the Gini
-index (for bagging classification trees). In the case of bagging
-regression trees, we can record the total amount that the MSE is
-decreased due to splits over a given predictor, averaged over all \( B \) possible
-trees. A large value indicates an important predictor. Similarly, in
-the context of bagging classification trees, we can add up the total
-amount that the Gini index  is decreased by splits over a given
-predictor, averaged over all \( B \) trees.
+However, by aggregating many decision trees, using methods like bagging, random forests, and boosting, the predictive performance of trees can be substantially improved.
 
 
    
 
    
@@ -234,6 +213,7 @@ predictor, averaged over all \( B \) trees.
   
  • 31
    • 
   
  • 32
    • 
   
  • 33
    • 
+  
  • 34
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs026.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs026.html
index 1a7a4d4ff..585aa4cfd 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs026.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs026.html
@@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source
               ('A Classification Tree', 2, None, '___sec12'),
               ('Growing a classification tree', 2, None, '___sec13'),
               ('Classification tree, how to split nodes', 2, None, '___sec14'),
-              ('Entropy and the ID3 algorithm', 2, None, '___sec15'),
-              ('Implementing the ID3 Algorithm', 2, None, '___sec16'),
+              ('The CART (Classification and Regression Tree) algorithm',
+               2,
+               None,
+               '___sec15'),
+              ('Entropy and the ID3 algorithm', 2, None, '___sec16'),
+              ('Implementing the ID3 Algorithm', 2, None, '___sec17'),
               ('Cancer Data again now with Decision Trees',
                2,
                None,
-               '___sec17'),
-              ('Another example, the moons again', 2, None, '___sec18'),
-              ('Playing around with regions', 2, None, '___sec19'),
-              ('Regression trees', 2, None, '___sec20'),
-              ('Final regressor code', 2, None, '___sec21'),
-              ('Pros and cons of trees, pros', 2, None, '___sec22'),
-              ('Disadvantages', 2, None, '___sec23'),
-              ('Bagging', 2, None, '___sec24'),
-              ('Simple example, head or tail', 2, None, '___sec25'),
-              ('Random forests', 2, None, '___sec26'),
-              ('A simple scikit-learn example', 2, None, '___sec27'),
-              ('Please, not the moons again!', 2, None, '___sec28'),
-              ('Bagging examples', 2, None, '___sec29'),
-              ('Then random forests', 2, None, '___sec30'),
-              ('Boosting and more', 2, None, '___sec31')]}
+               '___sec18'),
+              ('Another example, the moons again', 2, None, '___sec19'),
+              ('Playing around with regions', 2, None, '___sec20'),
+              ('Regression trees', 2, None, '___sec21'),
+              ('Final regressor code', 2, None, '___sec22'),
+              ('Pros and cons of trees, pros', 2, None, '___sec23'),
+              ('Disadvantages', 2, None, '___sec24'),
+              ('Bagging', 2, None, '___sec25'),
+              ('Simple example, head or tail', 2, None, '___sec26'),
+              ('Random forests', 2, None, '___sec27'),
+              ('A simple scikit-learn example', 2, None, '___sec28'),
+              ('Please, not the moons again!', 2, None, '___sec29'),
+              ('Bagging examples', 2, None, '___sec30'),
+              ('Then random forests', 2, None, '___sec31'),
+              ('Boosting and more', 2, None, '___sec32')]}
 end of tocinfo -->
 
 
@@ -139,23 +143,24 @@ MathJax.Hub.Config({
       
  • A Classification Tree
    • 
       
  • Growing a classification tree
    • 
       
  • Classification tree, how to split nodes
    • 
-      
  • Entropy and the ID3 algorithm
    • 
-      
  • Implementing the ID3 Algorithm
    • 
-      
  • Cancer Data again now with Decision Trees
    • 
-      
  • Another example, the moons again
    • 
-      
  • Playing around with regions
    • 
-      
  • Regression trees
    • 
-      
  • Final regressor code
    • 
-      
  • Pros and cons of trees, pros
    • 
-      
  • Disadvantages
    • 
-      
  • Bagging
    • 
-      
  • Simple example, head or tail
    • 
-      
  • Random forests
    • 
-      
  • A simple scikit-learn example
    • 
-      
  • Please, not the moons again!
    • 
-      
  • Bagging examples
    • 
-      
  • Then random forests
    • 
-      
  • Boosting and more
    • 
+      
  • The CART (Classification and Regression Tree) algorithm
    • 
+      
  • Entropy and the ID3 algorithm
    • 
+      
  • Implementing the ID3 Algorithm
    • 
+      
  • Cancer Data again now with Decision Trees
    • 
+      
  • Another example, the moons again
    • 
+      
  • Playing around with regions
    • 
+      
  • Regression trees
    • 
+      
  • Final regressor code
    • 
+      
  • Pros and cons of trees, pros
    • 
+      
  • Disadvantages
    • 
+      
  • Bagging
    • 
+      
  • Simple example, head or tail
    • 
+      
  • Random forests
    • 
+      
  • A simple scikit-learn example
    • 
+      
  • Please, not the moons again!
    • 
+      
  • Bagging examples
    • 
+      
  • Then random forests
    • 
+      
  • Boosting and more
    • 
 
         
       
@@ -171,23 +176,46 @@ MathJax.Hub.Config({
 
 
 
-
    Simple example, head or tail 
    
-
    
+
    Bagging 
    
+
+
    
+The plain decision trees suffer from high
+variance. This means that if we split the training data into two parts
+at random, and fit a decision tree to both halves, the results that we
+get could be quite different. In contrast, a procedure with low
+variance will yield similar results if applied repeatedly to distinct
+data sets; linear regression tends to have low variance, if the ratio
+of \( n \) to \( p \) is moderately large.
+
+
    
+Bootstrap aggregation, or just bagging, is a
+general-purpose procedure for reducing the variance of a statistical
+learning method.
+
+
    
+Bagging typically results in improved accuracy
+over prediction using a single tree. Unfortunately, however, it can be
+difficult to interpret the resulting model. Recall that one of the
+advantages of decision trees is the attractive and easily interpreted
+diagram that results.
+
+
    
+However, when we bag a large number of trees, it is no longer
+possible to represent the resulting statistical learning procedure
+using a single tree, and it is no longer clear which variables are
+most important to the procedure. Thus, bagging improves prediction
+accuracy at the expense of interpretability.  Although the collection
+of bagged trees is much more difficult to interpret than a single
+tree, one can obtain an overall summary of the importance of each
+predictor using the MSE (for bagging regression trees) or the Gini
+index (for bagging classification trees). In the case of bagging
+regression trees, we can record the total amount that the MSE is
+decreased due to splits over a given predictor, averaged over all \( B \) possible
+trees. A large value indicates an important predictor. Similarly, in
+the context of bagging classification trees, we can add up the total
+amount that the Gini index  is decreased by splits over a given
+predictor, averaged over all \( B \) trees.
 
-
-
    heads_proba = 0.51
-coin_tosses = (np.random.rand(10000, 10) < heads_proba).astype(np.int32)
-cumulative_heads_ratio = np.cumsum(coin_tosses, axis=0) / np.arange(1, 10001).reshape(-1, 1)
-plt.figure(figsize=(8,3.5))
-plt.plot(cumulative_heads_ratio)
-plt.plot([0, 10000], [0.51, 0.51], "k--", linewidth=2, label="51%")
-plt.plot([0, 10000], [0.5, 0.5], "k-", label="50%")
-plt.xlabel("Number of coin tosses")
-plt.ylabel("Heads ratio")
-plt.legend(loc="lower right")
-plt.axis([0, 10000, 0.42, 0.58])
-plt.show()
-
    
 
    
 
    
 
@@ -210,6 +238,7 @@ plt.show()
   
  • 31
    • 
   
  • 32
    • 
   
  • 33
    • 
+  
  • 34
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs027.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs027.html
index 409250a3e..7c8f43f65 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs027.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs027.html
@@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source
               ('A Classification Tree', 2, None, '___sec12'),
               ('Growing a classification tree', 2, None, '___sec13'),
               ('Classification tree, how to split nodes', 2, None, '___sec14'),
-              ('Entropy and the ID3 algorithm', 2, None, '___sec15'),
-              ('Implementing the ID3 Algorithm', 2, None, '___sec16'),
+              ('The CART (Classification and Regression Tree) algorithm',
+               2,
+               None,
+               '___sec15'),
+              ('Entropy and the ID3 algorithm', 2, None, '___sec16'),
+              ('Implementing the ID3 Algorithm', 2, None, '___sec17'),
               ('Cancer Data again now with Decision Trees',
                2,
                None,
-               '___sec17'),
-              ('Another example, the moons again', 2, None, '___sec18'),
-              ('Playing around with regions', 2, None, '___sec19'),
-              ('Regression trees', 2, None, '___sec20'),
-              ('Final regressor code', 2, None, '___sec21'),
-              ('Pros and cons of trees, pros', 2, None, '___sec22'),
-              ('Disadvantages', 2, None, '___sec23'),
-              ('Bagging', 2, None, '___sec24'),
-              ('Simple example, head or tail', 2, None, '___sec25'),
-              ('Random forests', 2, None, '___sec26'),
-              ('A simple scikit-learn example', 2, None, '___sec27'),
-              ('Please, not the moons again!', 2, None, '___sec28'),
-              ('Bagging examples', 2, None, '___sec29'),
-              ('Then random forests', 2, None, '___sec30'),
-              ('Boosting and more', 2, None, '___sec31')]}
+               '___sec18'),
+              ('Another example, the moons again', 2, None, '___sec19'),
+              ('Playing around with regions', 2, None, '___sec20'),
+              ('Regression trees', 2, None, '___sec21'),
+              ('Final regressor code', 2, None, '___sec22'),
+              ('Pros and cons of trees, pros', 2, None, '___sec23'),
+              ('Disadvantages', 2, None, '___sec24'),
+              ('Bagging', 2, None, '___sec25'),
+              ('Simple example, head or tail', 2, None, '___sec26'),
+              ('Random forests', 2, None, '___sec27'),
+              ('A simple scikit-learn example', 2, None, '___sec28'),
+              ('Please, not the moons again!', 2, None, '___sec29'),
+              ('Bagging examples', 2, None, '___sec30'),
+              ('Then random forests', 2, None, '___sec31'),
+              ('Boosting and more', 2, None, '___sec32')]}
 end of tocinfo -->
 
 
@@ -139,23 +143,24 @@ MathJax.Hub.Config({
       
  • A Classification Tree
    • 
       
  • Growing a classification tree
    • 
       
  • Classification tree, how to split nodes
    • 
-      
  • Entropy and the ID3 algorithm
    • 
-      
  • Implementing the ID3 Algorithm
    • 
-      
  • Cancer Data again now with Decision Trees
    • 
-      
  • Another example, the moons again
    • 
-      
  • Playing around with regions
    • 
-      
  • Regression trees
    • 
-      
  • Final regressor code
    • 
-      
  • Pros and cons of trees, pros
    • 
-      
  • Disadvantages
    • 
-      
  • Bagging
    • 
-      
  • Simple example, head or tail
    • 
-      
  • Random forests
    • 
-      
  • A simple scikit-learn example
    • 
-      
  • Please, not the moons again!
    • 
-      
  • Bagging examples
    • 
-      
  • Then random forests
    • 
-      
  • Boosting and more
    • 
+      
  • The CART (Classification and Regression Tree) algorithm
    • 
+      
  • Entropy and the ID3 algorithm
    • 
+      
  • Implementing the ID3 Algorithm
    • 
+      
  • Cancer Data again now with Decision Trees
    • 
+      
  • Another example, the moons again
    • 
+      
  • Playing around with regions
    • 
+      
  • Regression trees
    • 
+      
  • Final regressor code
    • 
+      
  • Pros and cons of trees, pros
    • 
+      
  • Disadvantages
    • 
+      
  • Bagging
    • 
+      
  • Simple example, head or tail
    • 
+      
  • Random forests
    • 
+      
  • A simple scikit-learn example
    • 
+      
  • Please, not the moons again!
    • 
+      
  • Bagging examples
    • 
+      
  • Then random forests
    • 
+      
  • Boosting and more
    • 
 
         
       
@@ -171,47 +176,23 @@ MathJax.Hub.Config({
 
 
 
-
    Random forests 
    
-
+
    Simple example, head or tail 
    
 
    
-Random forests provide an improvement over bagged trees by way of a
-small tweak that decorrelates the trees.
-
-
    
-As in bagging, we build a
-number of decision trees on bootstrapped training samples. But when
-building these decision trees, each time a split in a tree is
-considered, a random sample of \( m \) predictors is chosen as split
-candidates from the full set of \( p \) predictors. The split is allowed to
-use only one of those \( m \) predictors.
-
-
    
-A fresh sample of \( m \) predictors is
-taken at each split, and typically we choose 
-
-$$
-m\approx \sqrt{p}.
-$$
-
-
    
-In building a random forest, at
-each split in the tree, the algorithm is not even allowed to consider
-a majority of the available predictors.
-
-
    
-The reason for this is rather clever. Suppose that there is one very
-strong predictor in the data set, along with a number of other
-moderately strong predictors. Then in the collection of bagged
-variable importance random forest trees, most or all of the trees will
-use this strong predictor in the top split. Consequently, all of the
-bagged trees will look quite similar to each other. Hence the
-predictions from the bagged trees will be highly correlated.
-Unfortunately, averaging many highly correlated quantities does not
-lead to as large of a reduction in variance as averaging many
-uncorrelated quanti- ties. In particular, this means that bagging will
-not lead to a substantial reduction in variance over a single tree in
-this setting.
 
+
+
    heads_proba = 0.51
+coin_tosses = (np.random.rand(10000, 10) < heads_proba).astype(np.int32)
+cumulative_heads_ratio = np.cumsum(coin_tosses, axis=0) / np.arange(1, 10001).reshape(-1, 1)
+plt.figure(figsize=(8,3.5))
+plt.plot(cumulative_heads_ratio)
+plt.plot([0, 10000], [0.51, 0.51], "k--", linewidth=2, label="51%")
+plt.plot([0, 10000], [0.5, 0.5], "k-", label="50%")
+plt.xlabel("Number of coin tosses")
+plt.ylabel("Heads ratio")
+plt.legend(loc="lower right")
+plt.axis([0, 10000, 0.42, 0.58])
+plt.show()
+
    
 
    
 
    
 
@@ -233,6 +214,7 @@ this setting.
   
  • 31
    • 
   
  • 32
    • 
   
  • 33
    • 
+  
  • 34
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs028.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs028.html
index d009c481c..6dcc724bf 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs028.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs028.html
@@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source
               ('A Classification Tree', 2, None, '___sec12'),
               ('Growing a classification tree', 2, None, '___sec13'),
               ('Classification tree, how to split nodes', 2, None, '___sec14'),
-              ('Entropy and the ID3 algorithm', 2, None, '___sec15'),
-              ('Implementing the ID3 Algorithm', 2, None, '___sec16'),
+              ('The CART (Classification and Regression Tree) algorithm',
+               2,
+               None,
+               '___sec15'),
+              ('Entropy and the ID3 algorithm', 2, None, '___sec16'),
+              ('Implementing the ID3 Algorithm', 2, None, '___sec17'),
               ('Cancer Data again now with Decision Trees',
                2,
                None,
-               '___sec17'),
-              ('Another example, the moons again', 2, None, '___sec18'),
-              ('Playing around with regions', 2, None, '___sec19'),
-              ('Regression trees', 2, None, '___sec20'),
-              ('Final regressor code', 2, None, '___sec21'),
-              ('Pros and cons of trees, pros', 2, None, '___sec22'),
-              ('Disadvantages', 2, None, '___sec23'),
-              ('Bagging', 2, None, '___sec24'),
-              ('Simple example, head or tail', 2, None, '___sec25'),
-              ('Random forests', 2, None, '___sec26'),
-              ('A simple scikit-learn example', 2, None, '___sec27'),
-              ('Please, not the moons again!', 2, None, '___sec28'),
-              ('Bagging examples', 2, None, '___sec29'),
-              ('Then random forests', 2, None, '___sec30'),
-              ('Boosting and more', 2, None, '___sec31')]}
+               '___sec18'),
+              ('Another example, the moons again', 2, None, '___sec19'),
+              ('Playing around with regions', 2, None, '___sec20'),
+              ('Regression trees', 2, None, '___sec21'),
+              ('Final regressor code', 2, None, '___sec22'),
+              ('Pros and cons of trees, pros', 2, None, '___sec23'),
+              ('Disadvantages', 2, None, '___sec24'),
+              ('Bagging', 2, None, '___sec25'),
+              ('Simple example, head or tail', 2, None, '___sec26'),
+              ('Random forests', 2, None, '___sec27'),
+              ('A simple scikit-learn example', 2, None, '___sec28'),
+              ('Please, not the moons again!', 2, None, '___sec29'),
+              ('Bagging examples', 2, None, '___sec30'),
+              ('Then random forests', 2, None, '___sec31'),
+              ('Boosting and more', 2, None, '___sec32')]}
 end of tocinfo -->
 
 
@@ -139,23 +143,24 @@ MathJax.Hub.Config({
       
  • A Classification Tree
    • 
       
  • Growing a classification tree
    • 
       
  • Classification tree, how to split nodes
    • 
-      
  • Entropy and the ID3 algorithm
    • 
-      
  • Implementing the ID3 Algorithm
    • 
-      
  • Cancer Data again now with Decision Trees
    • 
-      
  • Another example, the moons again
    • 
-      
  • Playing around with regions
    • 
-      
  • Regression trees
    • 
-      
  • Final regressor code
    • 
-      
  • Pros and cons of trees, pros
    • 
-      
  • Disadvantages
    • 
-      
  • Bagging
    • 
-      
  • Simple example, head or tail
    • 
-      
  • Random forests
    • 
-      
  • A simple scikit-learn example
    • 
-      
  • Please, not the moons again!
    • 
-      
  • Bagging examples
    • 
-      
  • Then random forests
    • 
-      
  • Boosting and more
    • 
+      
  • The CART (Classification and Regression Tree) algorithm
    • 
+      
  • Entropy and the ID3 algorithm
    • 
+      
  • Implementing the ID3 Algorithm
    • 
+      
  • Cancer Data again now with Decision Trees
    • 
+      
  • Another example, the moons again
    • 
+      
  • Playing around with regions
    • 
+      
  • Regression trees
    • 
+      
  • Final regressor code
    • 
+      
  • Pros and cons of trees, pros
    • 
+      
  • Disadvantages
    • 
+      
  • Bagging
    • 
+      
  • Simple example, head or tail
    • 
+      
  • Random forests
    • 
+      
  • A simple scikit-learn example
    • 
+      
  • Please, not the moons again!
    • 
+      
  • Bagging examples
    • 
+      
  • Then random forests
    • 
+      
  • Boosting and more
    • 
 
         
       
@@ -171,21 +176,47 @@ MathJax.Hub.Config({
 
 
 
-
    A simple scikit-learn example 
    
-
    
+
    Random forests 
    
+
+
    
+Random forests provide an improvement over bagged trees by way of a
+small tweak that decorrelates the trees.
+
+
    
+As in bagging, we build a
+number of decision trees on bootstrapped training samples. But when
+building these decision trees, each time a split in a tree is
+considered, a random sample of \( m \) predictors is chosen as split
+candidates from the full set of \( p \) predictors. The split is allowed to
+use only one of those \( m \) predictors.
+
+
    
+A fresh sample of \( m \) predictors is
+taken at each split, and typically we choose 
+
+$$
+m\approx \sqrt{p}.
+$$
+
+
    
+In building a random forest, at
+each split in the tree, the algorithm is not even allowed to consider
+a majority of the available predictors.
+
+
    
+The reason for this is rather clever. Suppose that there is one very
+strong predictor in the data set, along with a number of other
+moderately strong predictors. Then in the collection of bagged
+variable importance random forest trees, most or all of the trees will
+use this strong predictor in the top split. Consequently, all of the
+bagged trees will look quite similar to each other. Hence the
+predictions from the bagged trees will be highly correlated.
+Unfortunately, averaging many highly correlated quantities does not
+lead to as large of a reduction in variance as averaging many
+uncorrelated quanti- ties. In particular, this means that bagging will
+not lead to a substantial reduction in variance over a single tree in
+this setting.
 
-
-
    from sklearn.ensemble import RandomForestClassifier
-from sklearn.preprocessing import LabelEncoder
-from sklearn.model_selection import cross_validate
-# Data set not specificied
-X = dataset.XXX
-Y = dataset.YYY
-#Instantiate the model with 100 trees and entropy as splitting criteria
-Random_Forest_model = RandomForestClassifier(n_estimators=100,criterion="entropy")
-#Cross validation
-accuracy = cross_validate(Random_Forest_model,X,Y,cv=10)['test_score']
-
    
 
    
 
    
 
@@ -206,6 +237,7 @@ accuracy = cross_validate(Random_Forest_mode
   
  • 31
    • 
   
  • 32
    • 
   
  • 33
    • 
+  
  • 34
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs029.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs029.html
index f5146b5a8..c7afc7f08 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs029.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs029.html
@@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source
               ('A Classification Tree', 2, None, '___sec12'),
               ('Growing a classification tree', 2, None, '___sec13'),
               ('Classification tree, how to split nodes', 2, None, '___sec14'),
-              ('Entropy and the ID3 algorithm', 2, None, '___sec15'),
-              ('Implementing the ID3 Algorithm', 2, None, '___sec16'),
+              ('The CART (Classification and Regression Tree) algorithm',
+               2,
+               None,
+               '___sec15'),
+              ('Entropy and the ID3 algorithm', 2, None, '___sec16'),
+              ('Implementing the ID3 Algorithm', 2, None, '___sec17'),
               ('Cancer Data again now with Decision Trees',
                2,
                None,
-               '___sec17'),
-              ('Another example, the moons again', 2, None, '___sec18'),
-              ('Playing around with regions', 2, None, '___sec19'),
-              ('Regression trees', 2, None, '___sec20'),
-              ('Final regressor code', 2, None, '___sec21'),
-              ('Pros and cons of trees, pros', 2, None, '___sec22'),
-              ('Disadvantages', 2, None, '___sec23'),
-              ('Bagging', 2, None, '___sec24'),
-              ('Simple example, head or tail', 2, None, '___sec25'),
-              ('Random forests', 2, None, '___sec26'),
-              ('A simple scikit-learn example', 2, None, '___sec27'),
-              ('Please, not the moons again!', 2, None, '___sec28'),
-              ('Bagging examples', 2, None, '___sec29'),
-              ('Then random forests', 2, None, '___sec30'),
-              ('Boosting and more', 2, None, '___sec31')]}
+               '___sec18'),
+              ('Another example, the moons again', 2, None, '___sec19'),
+              ('Playing around with regions', 2, None, '___sec20'),
+              ('Regression trees', 2, None, '___sec21'),
+              ('Final regressor code', 2, None, '___sec22'),
+              ('Pros and cons of trees, pros', 2, None, '___sec23'),
+              ('Disadvantages', 2, None, '___sec24'),
+              ('Bagging', 2, None, '___sec25'),
+              ('Simple example, head or tail', 2, None, '___sec26'),
+              ('Random forests', 2, None, '___sec27'),
+              ('A simple scikit-learn example', 2, None, '___sec28'),
+              ('Please, not the moons again!', 2, None, '___sec29'),
+              ('Bagging examples', 2, None, '___sec30'),
+              ('Then random forests', 2, None, '___sec31'),
+              ('Boosting and more', 2, None, '___sec32')]}
 end of tocinfo -->
 
 
@@ -139,23 +143,24 @@ MathJax.Hub.Config({
       
  • A Classification Tree
    • 
       
  • Growing a classification tree
    • 
       
  • Classification tree, how to split nodes
    • 
-      
  • Entropy and the ID3 algorithm
    • 
-      
  • Implementing the ID3 Algorithm
    • 
-      
  • Cancer Data again now with Decision Trees
    • 
-      
  • Another example, the moons again
    • 
-      
  • Playing around with regions
    • 
-      
  • Regression trees
    • 
-      
  • Final regressor code
    • 
-      
  • Pros and cons of trees, pros
    • 
-      
  • Disadvantages
    • 
-      
  • Bagging
    • 
-      
  • Simple example, head or tail
    • 
-      
  • Random forests
    • 
-      
  • A simple scikit-learn example
    • 
-      
  • Please, not the moons again!
    • 
-      
  • Bagging examples
    • 
-      
  • Then random forests
    • 
-      
  • Boosting and more
    • 
+      
  • The CART (Classification and Regression Tree) algorithm
    • 
+      
  • Entropy and the ID3 algorithm
    • 
+      
  • Implementing the ID3 Algorithm
    • 
+      
  • Cancer Data again now with Decision Trees
    • 
+      
  • Another example, the moons again
    • 
+      
  • Playing around with regions
    • 
+      
  • Regression trees
    • 
+      
  • Final regressor code
    • 
+      
  • Pros and cons of trees, pros
    • 
+      
  • Disadvantages
    • 
+      
  • Bagging
    • 
+      
  • Simple example, head or tail
    • 
+      
  • Random forests
    • 
+      
  • A simple scikit-learn example
    • 
+      
  • Please, not the moons again!
    • 
+      
  • Bagging examples
    • 
+      
  • Then random forests
    • 
+      
  • Boosting and more
    • 
 
         
       
@@ -171,60 +176,20 @@ MathJax.Hub.Config({
 
 
 
-
    Please, not the moons again! 
    
+
    A simple scikit-learn example 
    
 
    
 
 
-
    from sklearn.model_selection import train_test_split
-from sklearn.datasets import make_moons
-
-X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
-X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.ensemble import VotingClassifier
-from sklearn.linear_model import LogisticRegression
-from sklearn.svm import SVC
-
-log_clf = LogisticRegression(random_state=42)
-rnd_clf = RandomForestClassifier(random_state=42)
-svm_clf = SVC(random_state=42)
-
-voting_clf = VotingClassifier(
-    estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
-    voting='hard')
-voting_clf.fit(X_train, y_train)
-
    
-
    
-
-
-
    from sklearn.metrics import accuracy_score
-
-for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
-    clf.fit(X_train, y_train)
-    y_pred = clf.predict(X_test)
-    print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
-
    
-
    
-
-
-
    log_clf = LogisticRegression(random_state=42)
-rnd_clf = RandomForestClassifier(random_state=42)
-svm_clf = SVC(probability=True, random_state=42)
-
-voting_clf = VotingClassifier(
-    estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
-    voting='soft')
-voting_clf.fit(X_train, y_train)
-
    
-
    
-
-
-
    from sklearn.metrics import accuracy_score
-
-for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
-    clf.fit(X_train, y_train)
-    y_pred = clf.predict(X_test)
-    print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
+
    from sklearn.ensemble import RandomForestClassifier
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+# Data set not specificied
+X = dataset.XXX
+Y = dataset.YYY
+#Instantiate the model with 100 trees and entropy as splitting criteria
+Random_Forest_model = RandomForestClassifier(n_estimators=100,criterion="entropy")
+#Cross validation
+accuracy = cross_validate(Random_Forest_model,X,Y,cv=10)['test_score']
 
    
 
    
 
    
@@ -245,6 +210,7 @@ voting_clf.fit(X_train, y_train)
   
  • 31
    • 
   
  • 32
    • 
   
  • 33
    • 
+  
  • 34
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs030.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs030.html
index 5c6a146f2..233dda632 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs030.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs030.html
@@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source
               ('A Classification Tree', 2, None, '___sec12'),
               ('Growing a classification tree', 2, None, '___sec13'),
               ('Classification tree, how to split nodes', 2, None, '___sec14'),
-              ('Entropy and the ID3 algorithm', 2, None, '___sec15'),
-              ('Implementing the ID3 Algorithm', 2, None, '___sec16'),
+              ('The CART (Classification and Regression Tree) algorithm',
+               2,
+               None,
+               '___sec15'),
+              ('Entropy and the ID3 algorithm', 2, None, '___sec16'),
+              ('Implementing the ID3 Algorithm', 2, None, '___sec17'),
               ('Cancer Data again now with Decision Trees',
                2,
                None,
-               '___sec17'),
-              ('Another example, the moons again', 2, None, '___sec18'),
-              ('Playing around with regions', 2, None, '___sec19'),
-              ('Regression trees', 2, None, '___sec20'),
-              ('Final regressor code', 2, None, '___sec21'),
-              ('Pros and cons of trees, pros', 2, None, '___sec22'),
-              ('Disadvantages', 2, None, '___sec23'),
-              ('Bagging', 2, None, '___sec24'),
-              ('Simple example, head or tail', 2, None, '___sec25'),
-              ('Random forests', 2, None, '___sec26'),
-              ('A simple scikit-learn example', 2, None, '___sec27'),
-              ('Please, not the moons again!', 2, None, '___sec28'),
-              ('Bagging examples', 2, None, '___sec29'),
-              ('Then random forests', 2, None, '___sec30'),
-              ('Boosting and more', 2, None, '___sec31')]}
+               '___sec18'),
+              ('Another example, the moons again', 2, None, '___sec19'),
+              ('Playing around with regions', 2, None, '___sec20'),
+              ('Regression trees', 2, None, '___sec21'),
+              ('Final regressor code', 2, None, '___sec22'),
+              ('Pros and cons of trees, pros', 2, None, '___sec23'),
+              ('Disadvantages', 2, None, '___sec24'),
+              ('Bagging', 2, None, '___sec25'),
+              ('Simple example, head or tail', 2, None, '___sec26'),
+              ('Random forests', 2, None, '___sec27'),
+              ('A simple scikit-learn example', 2, None, '___sec28'),
+              ('Please, not the moons again!', 2, None, '___sec29'),
+              ('Bagging examples', 2, None, '___sec30'),
+              ('Then random forests', 2, None, '___sec31'),
+              ('Boosting and more', 2, None, '___sec32')]}
 end of tocinfo -->
 
 
@@ -139,23 +143,24 @@ MathJax.Hub.Config({
       
  • A Classification Tree
    • 
       
  • Growing a classification tree
    • 
       
  • Classification tree, how to split nodes
    • 
-      
  • Entropy and the ID3 algorithm
    • 
-      
  • Implementing the ID3 Algorithm
    • 
-      
  • Cancer Data again now with Decision Trees
    • 
-      
  • Another example, the moons again
    • 
-      
  • Playing around with regions
    • 
-      
  • Regression trees
    • 
-      
  • Final regressor code
    • 
-      
  • Pros and cons of trees, pros
    • 
-      
  • Disadvantages
    • 
-      
  • Bagging
    • 
-      
  • Simple example, head or tail
    • 
-      
  • Random forests
    • 
-      
  • A simple scikit-learn example
    • 
-      
  • Please, not the moons again!
    • 
-      
  • Bagging examples
    • 
-      
  • Then random forests
    • 
-      
  • Boosting and more
    • 
+      
  • The CART (Classification and Regression Tree) algorithm
    • 
+      
  • Entropy and the ID3 algorithm
    • 
+      
  • Implementing the ID3 Algorithm
    • 
+      
  • Cancer Data again now with Decision Trees
    • 
+      
  • Another example, the moons again
    • 
+      
  • Playing around with regions
    • 
+      
  • Regression trees
    • 
+      
  • Final regressor code
    • 
+      
  • Pros and cons of trees, pros
    • 
+      
  • Disadvantages
    • 
+      
  • Bagging
    • 
+      
  • Simple example, head or tail
    • 
+      
  • Random forests
    • 
+      
  • A simple scikit-learn example
    • 
+      
  • Please, not the moons again!
    • 
+      
  • Bagging examples
    • 
+      
  • Then random forests
    • 
+      
  • Boosting and more
    • 
 
         
       
@@ -171,63 +176,60 @@ MathJax.Hub.Config({
 
 
 
-
    Bagging examples 
    
-
+
    Please, not the moons again! 
    
 
    
 
 
-
    from sklearn.ensemble import BaggingClassifier
-from sklearn.tree import DecisionTreeClassifier
+
    from sklearn.model_selection import train_test_split
+from sklearn.datasets import make_moons
 
-bag_clf = BaggingClassifier(
-    DecisionTreeClassifier(random_state=42), n_estimators=500,
-    max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)
-bag_clf.fit(X_train, y_train)
-y_pred = bag_clf.predict(X_test)
+X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
+X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.ensemble import VotingClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.svm import SVC
+
+log_clf = LogisticRegression(random_state=42)
+rnd_clf = RandomForestClassifier(random_state=42)
+svm_clf = SVC(random_state=42)
+
+voting_clf = VotingClassifier(
+    estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
+    voting='hard')
+voting_clf.fit(X_train, y_train)
 
    
 
    
 
 
 
    from sklearn.metrics import accuracy_score
-print(accuracy_score(y_test, y_pred))
+
+for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
+    clf.fit(X_train, y_train)
+    y_pred = clf.predict(X_test)
+    print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
 
    
 
    
 
 
-
    tree_clf = DecisionTreeClassifier(random_state=42)
-tree_clf.fit(X_train, y_train)
-y_pred_tree = tree_clf.predict(X_test)
-print(accuracy_score(y_test, y_pred_tree))
+
    log_clf = LogisticRegression(random_state=42)
+rnd_clf = RandomForestClassifier(random_state=42)
+svm_clf = SVC(probability=True, random_state=42)
+
+voting_clf = VotingClassifier(
+    estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
+    voting='soft')
+voting_clf.fit(X_train, y_train)
 
    
 
    
 
 
-
    from matplotlib.colors import ListedColormap
+
    from sklearn.metrics import accuracy_score
 
-def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):
-    x1s = np.linspace(axes[0], axes[1], 100)
-    x2s = np.linspace(axes[2], axes[3], 100)
-    x1, x2 = np.meshgrid(x1s, x2s)
-    X_new = np.c_[x1.ravel(), x2.ravel()]
-    y_pred = clf.predict(X_new).reshape(x1.shape)
-    custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
-    plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
-    if contour:
-        custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
-        plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
-    plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha)
-    plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha)
-    plt.axis(axes)
-    plt.xlabel(r"$x_1$", fontsize=18)
-    plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
-plt.figure(figsize=(11,4))
-plt.subplot(121)
-plot_decision_boundary(tree_clf, X, y)
-plt.title("Decision Tree", fontsize=14)
-plt.subplot(122)
-plot_decision_boundary(bag_clf, X, y)
-plt.title("Decision Trees with Bagging", fontsize=14)
-plt.show()
+for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
+    clf.fit(X_train, y_train)
+    y_pred = clf.predict(X_test)
+    print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
 
    
 
    
 
    
@@ -247,6 +249,7 @@ plt.show()
   
  • 31
    • 
   
  • 32
    • 
   
  • 33
    • 
+  
  • 34
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs031.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs031.html
index 9b4825d3c..d85e0ae27 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs031.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs031.html
@@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source
               ('A Classification Tree', 2, None, '___sec12'),
               ('Growing a classification tree', 2, None, '___sec13'),
               ('Classification tree, how to split nodes', 2, None, '___sec14'),
-              ('Entropy and the ID3 algorithm', 2, None, '___sec15'),
-              ('Implementing the ID3 Algorithm', 2, None, '___sec16'),
+              ('The CART (Classification and Regression Tree) algorithm',
+               2,
+               None,
+               '___sec15'),
+              ('Entropy and the ID3 algorithm', 2, None, '___sec16'),
+              ('Implementing the ID3 Algorithm', 2, None, '___sec17'),
               ('Cancer Data again now with Decision Trees',
                2,
                None,
-               '___sec17'),
-              ('Another example, the moons again', 2, None, '___sec18'),
-              ('Playing around with regions', 2, None, '___sec19'),
-              ('Regression trees', 2, None, '___sec20'),
-              ('Final regressor code', 2, None, '___sec21'),
-              ('Pros and cons of trees, pros', 2, None, '___sec22'),
-              ('Disadvantages', 2, None, '___sec23'),
-              ('Bagging', 2, None, '___sec24'),
-              ('Simple example, head or tail', 2, None, '___sec25'),
-              ('Random forests', 2, None, '___sec26'),
-              ('A simple scikit-learn example', 2, None, '___sec27'),
-              ('Please, not the moons again!', 2, None, '___sec28'),
-              ('Bagging examples', 2, None, '___sec29'),
-              ('Then random forests', 2, None, '___sec30'),
-              ('Boosting and more', 2, None, '___sec31')]}
+               '___sec18'),
+              ('Another example, the moons again', 2, None, '___sec19'),
+              ('Playing around with regions', 2, None, '___sec20'),
+              ('Regression trees', 2, None, '___sec21'),
+              ('Final regressor code', 2, None, '___sec22'),
+              ('Pros and cons of trees, pros', 2, None, '___sec23'),
+              ('Disadvantages', 2, None, '___sec24'),
+              ('Bagging', 2, None, '___sec25'),
+              ('Simple example, head or tail', 2, None, '___sec26'),
+              ('Random forests', 2, None, '___sec27'),
+              ('A simple scikit-learn example', 2, None, '___sec28'),
+              ('Please, not the moons again!', 2, None, '___sec29'),
+              ('Bagging examples', 2, None, '___sec30'),
+              ('Then random forests', 2, None, '___sec31'),
+              ('Boosting and more', 2, None, '___sec32')]}
 end of tocinfo -->
 
 
@@ -139,23 +143,24 @@ MathJax.Hub.Config({
       
  • A Classification Tree
    • 
       
  • Growing a classification tree
    • 
       
  • Classification tree, how to split nodes
    • 
-      
  • Entropy and the ID3 algorithm
    • 
-      
  • Implementing the ID3 Algorithm
    • 
-      
  • Cancer Data again now with Decision Trees
    • 
-      
  • Another example, the moons again
    • 
-      
  • Playing around with regions
    • 
-      
  • Regression trees
    • 
-      
  • Final regressor code
    • 
-      
  • Pros and cons of trees, pros
    • 
-      
  • Disadvantages
    • 
-      
  • Bagging
    • 
-      
  • Simple example, head or tail
    • 
-      
  • Random forests
    • 
-      
  • A simple scikit-learn example
    • 
-      
  • Please, not the moons again!
    • 
-      
  • Bagging examples
    • 
-      
  • Then random forests
    • 
-      
  • Boosting and more
    • 
+      
  • The CART (Classification and Regression Tree) algorithm
    • 
+      
  • Entropy and the ID3 algorithm
    • 
+      
  • Implementing the ID3 Algorithm
    • 
+      
  • Cancer Data again now with Decision Trees
    • 
+      
  • Another example, the moons again
    • 
+      
  • Playing around with regions
    • 
+      
  • Regression trees
    • 
+      
  • Final regressor code
    • 
+      
  • Pros and cons of trees, pros
    • 
+      
  • Disadvantages
    • 
+      
  • Bagging
    • 
+      
  • Simple example, head or tail
    • 
+      
  • Random forests
    • 
+      
  • A simple scikit-learn example
    • 
+      
  • Please, not the moons again!
    • 
+      
  • Bagging examples
    • 
+      
  • Then random forests
    • 
+      
  • Boosting and more
    • 
 
         
       
@@ -171,24 +176,63 @@ MathJax.Hub.Config({
 
 
 
-
    Then random forests 
    
+
    Bagging examples 
    
+
 
    
 
 
-
    bag_clf = BaggingClassifier(
-    DecisionTreeClassifier(splitter="random", max_leaf_nodes=16, random_state=42),
-    n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1, random_state=42)
+
    from sklearn.ensemble import BaggingClassifier
+from sklearn.tree import DecisionTreeClassifier
+
+bag_clf = BaggingClassifier(
+    DecisionTreeClassifier(random_state=42), n_estimators=500,
+    max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)
+bag_clf.fit(X_train, y_train)
+y_pred = bag_clf.predict(X_test)
 
    
 
    
 
 
-
    bag_clf.fit(X_train, y_train)
-y_pred = bag_clf.predict(X_test)
-from sklearn.ensemble import RandomForestClassifier
-rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)
-rnd_clf.fit(X_train, y_train)
-y_pred_rf = rnd_clf.predict(X_test)
-np.sum(y_pred == y_pred_rf) / len(y_pred) 
+
    from sklearn.metrics import accuracy_score
+print(accuracy_score(y_test, y_pred))
+
    
+
    
+
+
+
    tree_clf = DecisionTreeClassifier(random_state=42)
+tree_clf.fit(X_train, y_train)
+y_pred_tree = tree_clf.predict(X_test)
+print(accuracy_score(y_test, y_pred_tree))
+
    
+
    
+
+
+
    from matplotlib.colors import ListedColormap
+
+def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):
+    x1s = np.linspace(axes[0], axes[1], 100)
+    x2s = np.linspace(axes[2], axes[3], 100)
+    x1, x2 = np.meshgrid(x1s, x2s)
+    X_new = np.c_[x1.ravel(), x2.ravel()]
+    y_pred = clf.predict(X_new).reshape(x1.shape)
+    custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
+    plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
+    if contour:
+        custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
+        plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
+    plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha)
+    plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha)
+    plt.axis(axes)
+    plt.xlabel(r"$x_1$", fontsize=18)
+    plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
+plt.figure(figsize=(11,4))
+plt.subplot(121)
+plot_decision_boundary(tree_clf, X, y)
+plt.title("Decision Tree", fontsize=14)
+plt.subplot(122)
+plot_decision_boundary(bag_clf, X, y)
+plt.title("Decision Trees with Bagging", fontsize=14)
+plt.show()
 
    
 
    
 
    
@@ -207,6 +251,7 @@ np.sum(y_pred =
   
  • 31
    • 
   
  • 32
    • 
   
  • 33
    • 
+  
  • 34
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-bs.html b/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
index ceb0e0248..8136086bf 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
@@ -67,26 +67,30 @@ Automatically generated HTML file from DocOnce source
               ('A Classification Tree', 2, None, '___sec12'),
               ('Growing a classification tree', 2, None, '___sec13'),
               ('Classification tree, how to split nodes', 2, None, '___sec14'),
-              ('Entropy and the ID3 algorithm', 2, None, '___sec15'),
-              ('Implementing the ID3 Algorithm', 2, None, '___sec16'),
+              ('The CART (Classification and Regression Tree) algorithm',
+               2,
+               None,
+               '___sec15'),
+              ('Entropy and the ID3 algorithm', 2, None, '___sec16'),
+              ('Implementing the ID3 Algorithm', 2, None, '___sec17'),
               ('Cancer Data again now with Decision Trees',
                2,
                None,
-               '___sec17'),
-              ('Another example, the moons again', 2, None, '___sec18'),
-              ('Playing around with regions', 2, None, '___sec19'),
-              ('Regression trees', 2, None, '___sec20'),
-              ('Final regressor code', 2, None, '___sec21'),
-              ('Pros and cons of trees, pros', 2, None, '___sec22'),
-              ('Disadvantages', 2, None, '___sec23'),
-              ('Bagging', 2, None, '___sec24'),
-              ('Simple example, head or tail', 2, None, '___sec25'),
-              ('Random forests', 2, None, '___sec26'),
-              ('A simple scikit-learn example', 2, None, '___sec27'),
-              ('Please, not the moons again!', 2, None, '___sec28'),
-              ('Bagging examples', 2, None, '___sec29'),
-              ('Then random forests', 2, None, '___sec30'),
-              ('Boosting and more', 2, None, '___sec31')]}
+               '___sec18'),
+              ('Another example, the moons again', 2, None, '___sec19'),
+              ('Playing around with regions', 2, None, '___sec20'),
+              ('Regression trees', 2, None, '___sec21'),
+              ('Final regressor code', 2, None, '___sec22'),
+              ('Pros and cons of trees, pros', 2, None, '___sec23'),
+              ('Disadvantages', 2, None, '___sec24'),
+              ('Bagging', 2, None, '___sec25'),
+              ('Simple example, head or tail', 2, None, '___sec26'),
+              ('Random forests', 2, None, '___sec27'),
+              ('A simple scikit-learn example', 2, None, '___sec28'),
+              ('Please, not the moons again!', 2, None, '___sec29'),
+              ('Bagging examples', 2, None, '___sec30'),
+              ('Then random forests', 2, None, '___sec31'),
+              ('Boosting and more', 2, None, '___sec32')]}
 end of tocinfo -->
 
 
@@ -139,23 +143,24 @@ MathJax.Hub.Config({
       
  • A Classification Tree
    • 
       
  • Growing a classification tree
    • 
       
  • Classification tree, how to split nodes
    • 
-      
  • Entropy and the ID3 algorithm
    • 
-      
  • Implementing the ID3 Algorithm
    • 
-      
  • Cancer Data again now with Decision Trees
    • 
-      
  • Another example, the moons again
    • 
-      
  • Playing around with regions
    • 
-      
  • Regression trees
    • 
-      
  • Final regressor code
    • 
-      
  • Pros and cons of trees, pros
    • 
-      
  • Disadvantages
    • 
-      
  • Bagging
    • 
-      
  • Simple example, head or tail
    • 
-      
  • Random forests
    • 
-      
  • A simple scikit-learn example
    • 
-      
  • Please, not the moons again!
    • 
-      
  • Bagging examples
    • 
-      
  • Then random forests
    • 
-      
  • Boosting and more
    • 
+      
  • The CART (Classification and Regression Tree) algorithm
    • 
+      
  • Entropy and the ID3 algorithm
    • 
+      
  • Implementing the ID3 Algorithm
    • 
+      
  • Cancer Data again now with Decision Trees
    • 
+      
  • Another example, the moons again
    • 
+      
  • Playing around with regions
    • 
+      
  • Regression trees
    • 
+      
  • Final regressor code
    • 
+      
  • Pros and cons of trees, pros
    • 
+      
  • Disadvantages
    • 
+      
  • Bagging
    • 
+      
  • Simple example, head or tail
    • 
+      
  • Random forests
    • 
+      
  • A simple scikit-learn example
    • 
+      
  • Please, not the moons again!
    • 
+      
  • Bagging examples
    • 
+      
  • Then random forests
    • 
+      
  • Boosting and more
    • 
 
         
       
@@ -190,7 +195,7 @@ MathJax.Hub.Config({
 
    [2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
    
 
    
 
    
-
    Oct 26, 2019
     
+
    Oct 29, 2019
     
 
    
 
    
 
@@ -214,7 +219,7 @@ MathJax.Hub.Config({
   
  • 9
    • 
   
  • 10
    • 
   
  • ...
    • 
-  
  • 33
    • 
+  
  • 34
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html b/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
index cf3dc3ebb..aca451881 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
@@ -148,7 +148,7 @@ MathJax.Hub.Config({
 
    [2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
    
 
    
 
     
    
-
    Oct 26, 2019
     
+
    Oct 29, 2019
     
 
    
 
    
 
@@ -627,7 +627,191 @@ $$
 
 
 
    
-
    Entropy and the ID3 algorithm 
    
+
    The CART (Classification and Regression Tree) algorithm 
    
+
+
    
+The above functions (gini, entropy and misclassification error) are important components of the so-called CART algorithm. We will discuss this algorithm first before we move on to the information gain algorithm ID3.
+
+
    
+
+
+
    from random import seed
+from random import randrange
+from csv import reader
+ 
+# Load a CSV file
+def load_csv(filename):
+	file = open(filename, "rb")
+	lines = reader(file)
+	dataset = list(lines)
+	return dataset
+ 
+# Convert string column to float
+def str_column_to_float(dataset, column):
+	for row in dataset:
+		row[column] = float(row[column].strip())
+ 
+# Split a dataset into k folds
+def cross_validation_split(dataset, n_folds):
+	dataset_split = list()
+	dataset_copy = list(dataset)
+	fold_size = int(len(dataset) / n_folds)
+	for i in range(n_folds):
+		fold = list()
+		while len(fold) < fold_size:
+			index = randrange(len(dataset_copy))
+			fold.append(dataset_copy.pop(index))
+		dataset_split.append(fold)
+	return dataset_split
+ 
+# Calculate accuracy percentage
+def accuracy_metric(actual, predicted):
+	correct = 0
+	for i in range(len(actual)):
+		if actual[i] == predicted[i]:
+			correct += 1
+	return correct / float(len(actual)) * 100.0
+ 
+# Evaluate an algorithm using a cross validation split
+def evaluate_algorithm(dataset, algorithm, n_folds, *args):
+	folds = cross_validation_split(dataset, n_folds)
+	scores = list()
+	for fold in folds:
+		train_set = list(folds)
+		train_set.remove(fold)
+		train_set = sum(train_set, [])
+		test_set = list()
+		for row in fold:
+			row_copy = list(row)
+			test_set.append(row_copy)
+			row_copy[-1] = None
+		predicted = algorithm(train_set, test_set, *args)
+		actual = [row[-1] for row in fold]
+		accuracy = accuracy_metric(actual, predicted)
+		scores.append(accuracy)
+	return scores
+ 
+# Split a dataset based on an attribute and an attribute value
+def test_split(index, value, dataset):
+	left, right = list(), list()
+	for row in dataset:
+		if row[index] < value:
+			left.append(row)
+		else:
+			right.append(row)
+	return left, right
+ 
+# Calculate the Gini index for a split dataset
+def gini_index(groups, classes):
+	# count all samples at split point
+	n_instances = float(sum([len(group) for group in groups]))
+	# sum weighted Gini index for each group
+	gini = 0.0
+	for group in groups:
+		size = float(len(group))
+		# avoid divide by zero
+		if size == 0:
+			continue
+		score = 0.0
+		# score the group based on the score for each class
+		for class_val in classes:
+			p = [row[-1] for row in group].count(class_val) / size
+			score += p * p
+		# weight the group score by its relative size
+		gini += (1.0 - score) * (size / n_instances)
+	return gini
+ 
+# Select the best split point for a dataset
+def get_split(dataset):
+	class_values = list(set(row[-1] for row in dataset))
+	b_index, b_value, b_score, b_groups = 999, 999, 999, None
+	for index in range(len(dataset[0])-1):
+		for row in dataset:
+			groups = test_split(index, row[index], dataset)
+			gini = gini_index(groups, class_values)
+			if gini < b_score:
+				b_index, b_value, b_score, b_groups = index, row[index], gini, groups
+	return {'index':b_index, 'value':b_value, 'groups':b_groups}
+ 
+# Create a terminal node value
+def to_terminal(group):
+	outcomes = [row[-1] for row in group]
+	return max(set(outcomes), key=outcomes.count)
+ 
+# Create child splits for a node or make terminal
+def split(node, max_depth, min_size, depth):
+	left, right = node['groups']
+	del(node['groups'])
+	# check for a no split
+	if not left or not right:
+		node['left'] = node['right'] = to_terminal(left + right)
+		return
+	# check for max depth
+	if depth >= max_depth:
+		node['left'], node['right'] = to_terminal(left), to_terminal(right)
+		return
+	# process left child
+	if len(left) <= min_size:
+		node['left'] = to_terminal(left)
+	else:
+		node['left'] = get_split(left)
+		split(node['left'], max_depth, min_size, depth+1)
+	# process right child
+	if len(right) <= min_size:
+		node['right'] = to_terminal(right)
+	else:
+		node['right'] = get_split(right)
+		split(node['right'], max_depth, min_size, depth+1)
+ 
+# Build a decision tree
+def build_tree(train, max_depth, min_size):
+	root = get_split(train)
+	split(root, max_depth, min_size, 1)
+	return root
+ 
+# Make a prediction with a decision tree
+def predict(node, row):
+	if row[node['index']] < node['value']:
+		if isinstance(node['left'], dict):
+			return predict(node['left'], row)
+		else:
+			return node['left']
+	else:
+		if isinstance(node['right'], dict):
+			return predict(node['right'], row)
+		else:
+			return node['right']
+ 
+# Classification and Regression Tree Algorithm
+def decision_tree(train, test, max_depth, min_size):
+	tree = build_tree(train, max_depth, min_size)
+	predictions = list()
+	for row in test:
+		prediction = predict(tree, row)
+		predictions.append(prediction)
+	return(predictions)
+ 
+# Test CART 
+seed(1)
+# load and prepare data
+filename = 'DataFiles/rideclass.csv'
+dataset = load_csv(filename)
+# convert string attributes to integers
+for i in range(len(dataset[0])):
+	str_column_to_float(dataset, i)
+# evaluate algorithm
+n_folds = 5
+max_depth = 5
+min_size = 10
+scores = evaluate_algorithm(dataset, decision_tree, n_folds, max_depth, min_size)
+print('Scores: %s' % scores)
+print('Mean Accuracy: %.3f%%' % (sum(scores)/float(len(scores))))
+
    
+
    
+
+
+
    
+
    Entropy and the ID3 algorithm 
    
 
 
    
 ID3, learns decision trees by constructing
@@ -664,7 +848,7 @@ attributes at each step while growing the tree.
 
 
 
    
-
    Implementing the ID3 Algorithm 
    
+
    Implementing the ID3 Algorithm 
    
 
 
    
 more text to come here, material presented during lecture Friday Oct 25.
@@ -672,7 +856,7 @@ attributes at each step while growing the tree.
 
 
 
    
-
    Cancer Data again now with Decision Trees 
    
+
    Cancer Data again now with Decision Trees 
    
 
    
 
 
@@ -722,7 +906,7 @@ deep_tree_clf.fit(X_train_scaled, y_train)
 
 
 
    
-
    Another example, the moons again 
    
+
    Another example, the moons again 
    
 
    
 
 
@@ -795,7 +979,7 @@ plt.show()
 
 
 
    
-
    Playing around with regions 
    
+
    Playing around with regions 
    
 
    
 
 
@@ -824,7 +1008,7 @@ plt.show()
 
 
 
    
-
    Regression trees 
    
+
    Regression trees 
    
 
    
 
 
@@ -847,7 +1031,7 @@ tree_reg.fit(X, y)
 
 
 
    
-
    Final regressor code 
    
+
    Final regressor code 
    
 
    
 
 
@@ -926,7 +1110,7 @@ plt.show()
 
 
 
    
-
    Pros and cons of trees, pros 
    
+
    Pros and cons of trees, pros