From 456a54ca01f04b13a81301cef51d3875fb4b6c02 Mon Sep 17 00:00:00 2001 From: mhjensen Date: Thu, 7 Nov 2019 06:56:53 +0100 Subject: [PATCH] added boost --- .../html/._DecisionTrees-bs000.html | 78 +++---- .../html/._DecisionTrees-bs001.html | 76 +++---- .../html/._DecisionTrees-bs002.html | 76 +++---- .../html/._DecisionTrees-bs003.html | 76 +++---- .../html/._DecisionTrees-bs004.html | 76 +++---- .../html/._DecisionTrees-bs005.html | 76 +++---- .../html/._DecisionTrees-bs006.html | 76 +++---- .../html/._DecisionTrees-bs007.html | 76 +++---- .../html/._DecisionTrees-bs008.html | 76 +++---- .../html/._DecisionTrees-bs009.html | 76 +++---- .../html/._DecisionTrees-bs010.html | 76 +++---- .../html/._DecisionTrees-bs011.html | 76 +++---- .../html/._DecisionTrees-bs012.html | 76 +++---- .../html/._DecisionTrees-bs013.html | 76 +++---- .../html/._DecisionTrees-bs014.html | 76 +++---- .../html/._DecisionTrees-bs015.html | 76 +++---- .../html/._DecisionTrees-bs016.html | 76 +++---- .../html/._DecisionTrees-bs017.html | 80 +++---- .../html/._DecisionTrees-bs018.html | 76 +++---- .../html/._DecisionTrees-bs019.html | 76 +++---- .../html/._DecisionTrees-bs020.html | 76 +++---- .../html/._DecisionTrees-bs021.html | 76 +++---- .../html/._DecisionTrees-bs022.html | 76 +++---- .../html/._DecisionTrees-bs023.html | 76 +++---- .../html/._DecisionTrees-bs024.html | 76 +++---- .../html/._DecisionTrees-bs025.html | 76 +++---- .../html/._DecisionTrees-bs026.html | 76 +++---- .../html/._DecisionTrees-bs027.html | 76 +++---- .../html/._DecisionTrees-bs028.html | 76 +++---- .../html/._DecisionTrees-bs029.html | 76 +++---- .../html/._DecisionTrees-bs030.html | 76 +++---- .../html/._DecisionTrees-bs031.html | 76 +++---- .../html/._DecisionTrees-bs032.html | 80 +++---- .../html/._DecisionTrees-bs033.html | 76 +++---- .../html/._DecisionTrees-bs034.html | 76 +++---- .../html/._DecisionTrees-bs035.html | 76 +++---- .../html/._DecisionTrees-bs036.html | 76 +++---- .../html/._DecisionTrees-bs037.html | 76 +++---- .../html/._DecisionTrees-bs038.html | 76 +++---- .../html/._DecisionTrees-bs039.html | 76 +++---- .../html/._DecisionTrees-bs040.html | 160 ++++++++------ .../html/._DecisionTrees-bs041.html | 126 ++++++----- .../html/._DecisionTrees-bs042.html | 158 +++++--------- .../html/._DecisionTrees-bs043.html | 153 +++++++++----- .../html/._DecisionTrees-bs044.html | 96 +++++---- .../html/._DecisionTrees-bs045.html | 139 ++++++++---- .../html/._DecisionTrees-bs046.html | 99 ++++----- .../html/._DecisionTrees-bs047.html | 117 +++++------ .../html/._DecisionTrees-bs048.html | 112 ++++++---- .../html/._DecisionTrees-bs049.html | 115 ++++------ .../html/._DecisionTrees-bs050.html | 117 +++++++---- .../html/._DecisionTrees-bs051.html | 102 +++++---- .../html/._DecisionTrees-bs052.html | 181 ++++++---------- .../html/._DecisionTrees-bs053.html | 187 ++++++++++------- .../DecisionTrees/html/DecisionTrees-bs.html | 78 +++---- .../html/DecisionTrees-reveal.html | 161 ++++++++++++-- .../html/DecisionTrees-solarized.html | 198 ++++++++++++++---- doc/pub/DecisionTrees/html/DecisionTrees.html | 198 ++++++++++++++---- .../DecisionTrees/ipynb/DecisionTrees.ipynb | 163 ++++++++++++-- .../ipynb/ipynb-DecisionTrees-src.tar.gz | Bin 294061 -> 294061 bytes .../pdf/DecisionTrees-minted.pdf | Bin 528594 -> 526615 bytes doc/src/DecisionTrees/DecisionTrees.do.txt | 131 +++++++++++- doc/src/DecisionTrees/Programs/bootstrap.py | 56 +++++ doc/src/DecisionTrees/Programs/rfbootstrap.py | 54 +++++ 64 files changed, 3525 insertions(+), 2426 deletions(-) create mode 100644 doc/src/DecisionTrees/Programs/bootstrap.py create mode 100644 doc/src/DecisionTrees/Programs/rfbootstrap.py diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs000.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs000.html index cd7951257..a15bd51fb 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs000.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs000.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -253,7 +259,7 @@ MathJax.Hub.Config({
    [2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University

    -

    Nov 6, 2019

    +

    Nov 7, 2019


    @@ -277,7 +283,7 @@ MathJax.Hub.Config({

  • 9
  • 10
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs001.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs001.html index 0756f3f60..b2cd9fb7b 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs001.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs001.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -282,7 +288,7 @@ given some assumptions, make predictions about the target feature value
  • 10
  • 11
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs002.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs002.html index b54a6221d..aa5980198 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs002.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs002.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -260,7 +266,7 @@ This tree was produced using the Wisconsin cancer data (discussed here as well,
  • 11
  • 12
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs003.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs003.html index 90142e15c..61fa8fbdb 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs003.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs003.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -268,7 +274,7 @@ node.
  • 12
  • 13
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs004.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs004.html index 5523306c3..64a653ebd 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs004.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs004.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -269,7 +275,7 @@ Then we are essentially done!
  • 13
  • 14
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs005.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs005.html index 6c480b0a7..8c1250e07 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs005.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs005.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -348,7 +354,7 @@ plt.show()
  • 14
  • 15
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs006.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs006.html index a68a5e269..16a66cb17 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs006.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs006.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -281,7 +287,7 @@ within box \( j \).
  • 15
  • 16
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs007.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs007.html index 073d3217f..7ce294c03 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs007.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs007.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -273,7 +279,7 @@ better tree in some future step.
  • 16
  • 17
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs008.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs008.html index fff86f1d8..753e842e5 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs008.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs008.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -306,7 +312,7 @@ region contains more than five observations.
  • 17
  • 18
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs009.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs009.html index 3054a7236..cd8fc3d5f 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs009.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs009.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -275,7 +281,7 @@ parameter \( \alpha \).
  • 18
  • 19
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs010.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs010.html index 06a27fb4d..413ea449d 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs010.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs010.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -288,7 +294,7 @@ subtree corresponding to \( \alpha \).
  • 19
  • 20
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs011.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs011.html index 6e736bbc3..040bfb450 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs011.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs011.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -284,7 +290,7 @@ MathJax.Hub.Config({
  • 20
  • 21
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs012.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs012.html index 695f10763..dfd9431c2 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs012.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs012.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -276,7 +282,7 @@ fall into that region.
  • 21
  • 22
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs013.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs013.html index a1136e70f..ec841c432 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs013.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs013.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -281,7 +287,7 @@ than is the classification error rate.
  • 22
  • 23
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs014.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs014.html index 639a87941..32747a5b5 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs014.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs014.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -307,7 +313,7 @@ $$
  • 23
  • 24
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs015.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs015.html index 01da17adf..8c075d6ca 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs015.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs015.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -298,7 +304,7 @@ os.system(cmd)
  • 24
  • 25
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs016.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs016.html index b486e8e6a..d10d8fe9f 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs016.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs016.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -289,7 +295,7 @@ os.system(cmd)
  • 25
  • 26
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs017.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs017.html index d1c8143d8..4099218b1 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs017.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs017.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -235,6 +241,8 @@ MathJax.Hub.Config({

    Algorithms for Setting up Decision Trees

    + +

    Two algorithms stand out in the set up of decision trees:

      @@ -242,7 +250,7 @@ Two algorithms stand out in the set up of decision trees:
    1. The ID3 algorithm based on the computation of the information gain for classification
    -We discuss both algorithms with applications here. The popular library -Scikit-Learn_ uses the CART algorithm. For classification problems you can use either the gini index or the entropy to split a tree in two branches. +We discuss both algorithms with applications here. The popular library Scikit-Learn uses the CART algorithm. For classification problems you can use either the gini index or the entropy to split a tree in two branches.

    @@ -270,7 +278,7 @@ We discuss both algorithms with applications here. The popular library -Scikit-L

  • 26
  • 27
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs018.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs018.html index 2a2418513..523dfd7a6 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs018.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs018.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -262,7 +268,7 @@ MathJax.Hub.Config({
  • 27
  • 28
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs019.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs019.html index ee57b6185..0986c5322 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs019.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs019.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -262,7 +268,7 @@ MathJax.Hub.Config({
  • 28
  • 29
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs020.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs020.html index 128133a1b..f239246f8 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs020.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs020.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -302,7 +308,7 @@ The table here summarizes the various attributes and
  • 29
  • 30
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs021.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs021.html index 8b3cdb618..83347ed34 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs021.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs021.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -333,7 +339,7 @@ os.system(cmd)
  • 30
  • 31
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs022.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs022.html index df9cb9a60..ae8bb366f 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs022.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs022.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -335,7 +341,7 @@ split = get_split(dataset)
  • 31
  • 32
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs023.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs023.html index 57c74b394..be1b6cdb0 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs023.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs023.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -293,7 +299,7 @@ attributes at each step while growing the tree.
  • 32
  • 33
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs024.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs024.html index 90f522b3b..ea13526c7 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs024.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs024.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -453,7 +459,7 @@ MathJax.Hub.Config({
  • 33
  • 34
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs025.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs025.html index 1c99d0a05..2dcfdb108 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs025.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs025.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -306,7 +312,7 @@ deep_tree_clf.fit(X_train_scaled, y_train)
  • 34
  • 35
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs026.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs026.html index 0b2a90953..c12b2aea9 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs026.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs026.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -329,7 +335,7 @@ plt.show()
  • 35
  • 36
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs027.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs027.html index 7de76160b..9a3fdcb8b 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs027.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs027.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -285,7 +291,7 @@ plt.show()
  • 36
  • 37
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs028.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs028.html index 3dc974db8..7cb8a8a9d 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs028.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs028.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -279,7 +285,7 @@ tree_reg.fit(X, y)
  • 37
  • 38
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs029.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs029.html index d1a4a97cb..d9412629b 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs029.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs029.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -335,7 +341,7 @@ plt.show()
  • 38
  • 39
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs030.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs030.html index c9cbd2f91..4a1e86086 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs030.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs030.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -271,7 +277,7 @@ MathJax.Hub.Config({
  • 39
  • 40
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs031.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs031.html index d0f98282a..fc659ead6 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs031.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs031.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -274,7 +280,7 @@ However, by aggregating many decision trees, using methods like bagging, random
  • 40
  • 41
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs032.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs032.html index fab9eb30f..256109b6c 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs032.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs032.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -234,7 +240,7 @@ MathJax.Hub.Config({ -

    From a Single Tree to Many Trees, Meet the Jungle of Methods

    +

    Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods

    As stated above and seen in many of the examples discussed here about @@ -250,7 +256,7 @@ machine learning algorithms or just use one of them to construct forests and jun

  • Voting classifiers
  • Bagging and Pasting
  • Random forests
    • -
  • Boosting methods
    • +
  • Boosting methods, from adaptive to Extreme Gradient Boosting (XGBoost)
    • We discuss these methods here. @@ -281,7 +287,7 @@ We discuss these methods here.
  • 41
  • 42
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs033.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs033.html index 264568a5f..7682c9683 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs033.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs033.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -265,7 +271,7 @@ MathJax.Hub.Config({
  • 42
  • 43
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs034.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs034.html index 78d22020a..dacb801bf 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs034.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs034.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -276,7 +282,7 @@ learning method.
  • 43
  • 44
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs035.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs035.html index c95f265ce..7f0156aa2 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs035.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs035.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -286,7 +292,7 @@ predictor, averaged over all \( B \) trees.
  • 44
  • 45
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs036.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs036.html index e44725d77..02a236fc7 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs036.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs036.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -277,7 +283,7 @@ plt.show()
  • 45
  • 46
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs037.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs037.html index 5c17f7f80..161d54c68 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs037.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs037.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -308,7 +314,7 @@ voting_clf.fit(X_train, y_train)
  • 46
  • 47
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs038.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs038.html index 06fff87ed..96e5c79e6 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs038.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs038.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -315,7 +321,7 @@ voting_clf.fit(X_train, y_train)
  • 47
  • 48
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs039.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs039.html index b381d8e1a..314a928c0 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs039.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs039.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -318,7 +324,7 @@ plt.show()
  • 48
  • 49
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs040.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs040.html index 9813a92be..e673f5578 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs040.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs040.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -234,47 +240,63 @@ MathJax.Hub.Config({ -

    Random forests

    - +

    Making our own Bagging with Bootstrap

    -Random forests provide an improvement over bagged trees by way of a -small tweak that decorrelates the trees. -

    -As in bagging, we build a -number of decision trees on bootstrapped training samples. But when -building these decision trees, each time a split in a tree is -considered, a random sample of \( m \) predictors is chosen as split -candidates from the full set of \( p \) predictors. The split is allowed to -use only one of those \( m \) predictors. + +

    import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
 
-
    
-A fresh sample of \( m \) predictors is
-taken at each split, and typically we choose 
 
-$$
-m\approx \sqrt{p}.
-$$
+np.random.seed(2018)
 
-
    
-In building a random forest, at
-each split in the tree, the algorithm is not even allowed to consider
-a majority of the available predictors.
+n = 40
+n_boostraps = 100
+maxdegree = 14
 
-
    
-The reason for this is rather clever. Suppose that there is one very
-strong predictor in the data set, along with a number of other
-moderately strong predictors. Then in the collection of bagged
-variable importance random forest trees, most or all of the trees will
-use this strong predictor in the top split. Consequently, all of the
-bagged trees will look quite similar to each other. Hence the
-predictions from the bagged trees will be highly correlated.
-Unfortunately, averaging many highly correlated quantities does not
-lead to as large of a reduction in variance as averaging many
-uncorrelated quanti- ties. In particular, this means that bagging will
-not lead to a substantial reduction in variance over a single tree in
-this setting.
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
 
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+    model = DecisionTreeRegressor(max_depth=5) 
+    y_pred = np.empty((y_test.shape[0], n_boostraps))
+    for i in range(n_boostraps):
+        x_, y_ = resample(X_train_scaled, y_train)
+        model.fit(x_, y_)
+        y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+    polydegree[degree] = degree
+    error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+    bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+    variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+    print('Polynomial degree:', degree)
+    print('Error:', error[degree])
+    print('Bias^2:', bias[degree])
+    print('Var:', variance[degree])
+    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+

    @@ -301,7 +323,7 @@ this setting.

  • 49
  • 50
  • ...
    • -
  • 56
    • +
  • 57
  • »
    • diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs041.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs041.html index ee5b1ad08..e51b1ff27 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs041.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs041.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -234,30 +240,48 @@ MathJax.Hub.Config({ -

    Random Forest Algorithm

    -The algorithm described here can be applied to both classification and regression problems. +

    Random forests

    -We will grow of forest of say \( M \) trees. +Random forests provide an improvement over bagged trees by way of a +small tweak that decorrelates the trees. -

      -
    1. For \( m=1:M \) we
      2. +

      +As in bagging, we build a +number of decision trees on bootstrapped training samples. But when +building these decision trees, each time a split in a tree is +considered, a random sample of \( m \) predictors is chosen as split +candidates from the full set of \( p \) predictors. The split is allowed to +use only one of those \( m \) predictors. -

      +

      +In building a random forest, at +each split in the tree, the algorithm is not even allowed to consider +a majority of the available predictors. -

    2. Output then the ensemble of trees \( \{T_m\}_1^{M} \) and make predictions for either a regression type of problem or a classification type of problem.
      3. -
    +

    +The reason for this is rather clever. Suppose that there is one very +strong predictor in the data set, along with a number of other +moderately strong predictors. Then in the collection of bagged +variable importance random forest trees, most or all of the trees will +use this strong predictor in the top split. Consequently, all of the +bagged trees will look quite similar to each other. Hence the +predictions from the bagged trees will be highly correlated. +Unfortunately, averaging many highly correlated quantities does not +lead to as large of a reduction in variance as averaging many +uncorrelated quanti- ties. In particular, this means that bagging will +not lead to a substantial reduction in variance over a single tree in +this setting. +

    diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs042.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs042.html index 4b3db8b2b..486a0c3b1 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs042.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs042.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -234,78 +240,30 @@ MathJax.Hub.Config({ -

    Random Forests Compared with other Methods on the Cancer Data

    +

    Random Forest Algorithm

    +The algorithm described here can be applied to both classification and regression problems. +

    +We will grow of forest of say \( M \) trees. - -

    import matplotlib.pyplot as plt
-import numpy as np
-from sklearn.model_selection import  train_test_split 
-from sklearn.datasets import load_breast_cancer
-from sklearn.svm import SVC
-from sklearn.linear_model import LogisticRegression
-from sklearn.tree import DecisionTreeClassifier
+
      
+
    1.  For \( m=1:M \) we
      2. 
 
-# Load the data
-cancer = load_breast_cancer()
+
        
+  
      •  Draw a bootstrap sample of from the training data organized in our \( \boldsymbol{X} \) matrix.
        • 
+  
      •  We grow then a random forest tree \( T_m \) based on the bootstrapped data by repeating the steps outlined till we reach the maximum node size is reached
        • 
 
-X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-print(X_train.shape)
-print(X_test.shape)
-# Logistic Regression
-logreg = LogisticRegression(solver='lbfgs')
-logreg.fit(X_train, y_train)
-print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
-# Support vector machine
-svm = SVC(gamma='auto', C=100)
-svm.fit(X_train, y_train)
-print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
-# Decision Trees
-deep_tree_clf = DecisionTreeClassifier(max_depth=None)
-deep_tree_clf.fit(X_train, y_train)
-print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
-#now scale the data
-from sklearn.preprocessing import StandardScaler
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-# Logistic Regression
-logreg.fit(X_train_scaled, y_train)
-print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Support Vector Machine
-svm.fit(X_train_scaled, y_train)
-print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Decision Trees
-deep_tree_clf.fit(X_train_scaled, y_train)
-print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
+
          
+   
        1.  we select \( m \le p \) varibales at random from the \( p \) predictors/features
          2. 
+   
        2.  pick the best split point among the \( m \) features using either the CART algorithm or the ID3 for classification and create a new node
          3. 
+   
        3.  split the node into daughter nodes
          4. 
+
        
 
+
      
 
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.preprocessing import LabelEncoder
-from sklearn.model_selection import cross_validate
-# Data set not specificied
-#Instantiate the model with 500 trees and entropy as splitting criteria
-Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy")
-Random_Forest_model.fit(X_train_scaled, y_train)
-#Cross validation
-accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score']
-print(accuracy)
-print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test)))
+
    2.  Output then the ensemble of trees \( \{T_m\}_1^{M} \) and make predictions for either a regression type of problem or a classification type of problem.
      3.  
+
    
 
-
-import scikitplot as skplt
-y_pred = Random_Forest_model.predict(X_test_scaled)
-skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
-#<matplotlib.axes._subplots.AxesSubplot object at 0x7fe967d64490>
-plt.show()
-y_probas = Random_Forest_model.predict_proba(X_test_scaled)
-skplt.metrics.plot_roc(y_test, y_probas)
-plt.show()
-skplt.metrics.plot_cumulative_gain(y_test, y_probas)
-plt.show()
-
    -

    diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs043.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs043.html index c8ef5cf7a..c4bc06499 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs043.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs043.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
  • Final regressor code
  • Pros and cons of trees, pros
  • Disadvantages
    • -
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • +
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
  • An Overview of Ensemble Methods
  • Bagging
  • More bagging
    • @@ -203,22 +208,23 @@ MathJax.Hub.Config({
  • Using the Voting Classifier
  • Please, not the moons again! Voting and Bagging
  • Now Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Feature Importance
    • -
  • Boosting, a Bird'e Eye
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • Figure to Illustrate the Iterative Classification Process
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples
    • -
  • Gradient Boots with Early Stopping
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Xgboost on the Cancer Data
    • +
  • Making our own Bagging with Bootstrap
    • +
  • Random forests
    • +
  • Random Forest Algorithm
    • +
  • Random Forests Compared with other Methods on the Cancer Data
    • +
  • Compare Bagging on Trees with Random Forests
    • +
  • Bootstrap with Random Forests Instead of a Single Tree
    • +
  • Boosting, a Bird'e Eye
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • Figure to Illustrate the Iterative Classification Process
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples
    • +
  • Gradient Boots with Early Stopping
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Xgboost on the Cancer Data
    • @@ -234,24 +240,75 @@ MathJax.Hub.Config({ -

    Compare Bagging on Trees with Random Forests

    +

    Random Forests Compared with other Methods on the Cancer Data

    -

    bag_clf = BaggingClassifier(
-    DecisionTreeClassifier(splitter="random", max_leaf_nodes=16, random_state=42),
-    n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1, random_state=42)
-
    -

    +

    import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import  train_test_split 
+from sklearn.datasets import load_breast_cancer
+from sklearn.svm import SVC
+from sklearn.linear_model import LogisticRegression
+from sklearn.tree import DecisionTreeClassifier
+
+# Load the data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+# Logistic Regression
+logreg = LogisticRegression(solver='lbfgs')
+logreg.fit(X_train, y_train)
+print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
+# Support vector machine
+svm = SVC(gamma='auto', C=100)
+svm.fit(X_train, y_train)
+print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
+# Decision Trees
+deep_tree_clf = DecisionTreeClassifier(max_depth=None)
+deep_tree_clf.fit(X_train, y_train)
+print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+# Logistic Regression
+logreg.fit(X_train_scaled, y_train)
+print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Support Vector Machine
+svm.fit(X_train_scaled, y_train)
+print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Decision Trees
+deep_tree_clf.fit(X_train_scaled, y_train)
+print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
+
 
-
-
    bag_clf.fit(X_train, y_train)
-y_pred = bag_clf.predict(X_test)
 from sklearn.ensemble import RandomForestClassifier
-rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)
-rnd_clf.fit(X_train, y_train)
-y_pred_rf = rnd_clf.predict(X_test)
-np.sum(y_pred == y_pred_rf) / len(y_pred) 
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+# Data set not specificied
+#Instantiate the model with 500 trees and entropy as splitting criteria
+Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy")
+Random_Forest_model.fit(X_train_scaled, y_train)
+#Cross validation
+accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score']
+print(accuracy)
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test)))
+
+
+import scikitplot as skplt
+y_pred = Random_Forest_model.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = Random_Forest_model.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
 
    
 
    
 
    
@@ -279,7 +336,7 @@ np.sum(y_pred =
   
  • 52
    • 
   
  • 53
    • 
   
  • ...
    • 
-  
  • 56
    • 
+  
  • 57
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs044.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs044.html
index c863be7df..2401d68f0 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs044.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs044.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
               ('Final regressor code', 2, None, '___sec28'),
               ('Pros and cons of trees, pros', 2, None, '___sec29'),
               ('Disadvantages', 2, None, '___sec30'),
-              ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+              ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+               'Boosting, Meet the Jungle of Methods',
                2,
                None,
                '___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
                None,
                '___sec37'),
               ('Now Bagging', 2, None, '___sec38'),
-              ('Random forests', 2, None, '___sec39'),
-              ('Random Forest Algorithm', 2, None, '___sec40'),
+              ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+              ('Random forests', 2, None, '___sec40'),
+              ('Random Forest Algorithm', 2, None, '___sec41'),
               ('Random Forests Compared with other Methods on the Cancer Data',
                2,
                None,
-               '___sec41'),
+               '___sec42'),
               ('Compare  Bagging on Trees with Random Forests',
                2,
                None,
-               '___sec42'),
-              ('Feature Importance', 2, None, '___sec43'),
-              ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+               '___sec43'),
+              ('Bootstrap with Random Forests Instead of a Single Tree',
+               2,
+               None,
+               '___sec44'),
+              ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
               ('Adaptive boosting: AdaBoost, Basic Algorithm',
                2,
                None,
-               '___sec45'),
-              ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+               '___sec46'),
+              ('Basic Steps of AdaBoost', 2, None, '___sec47'),
               ('Figure to Illustrate the Iterative Classification Process',
                2,
                None,
-               '___sec47'),
-              ('AdaBoost Examples', 2, None, '___sec48'),
-              ('Gradient boosting: Basics', 2, None, '___sec49'),
-              ('Gradient Boosting, algorithm', 2, None, '___sec50'),
-              ('Gradient Boosting, Examples', 2, None, '___sec51'),
-              ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
-              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
-              ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+               '___sec48'),
+              ('AdaBoost Examples', 2, None, '___sec49'),
+              ('Gradient boosting: Basics', 2, None, '___sec50'),
+              ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+              ('Gradient Boosting, Examples', 2, None, '___sec52'),
+              ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+              ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
 end of tocinfo -->
 
 
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
       
  • Final regressor code
    • 
       
  • Pros and cons of trees, pros
    • 
       
  • Disadvantages
    • 
-      
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • 
+      
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
    • 
       
  • An Overview of Ensemble Methods
    • 
       
  • Bagging
    • 
       
  • More bagging
    • 
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
       
  • Using the Voting Classifier
    • 
       
  • Please, not the moons again! Voting and Bagging
    • 
       
  • Now Bagging
    • 
-      
  • Random forests
    • 
-      
  • Random Forest Algorithm
    • 
-      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
-      
  • Compare  Bagging on Trees with Random Forests
    • 
-      
  • Feature Importance
    • 
-      
  • Boosting, a Bird'e Eye
    • 
-      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
-      
  • Basic Steps of AdaBoost
    • 
-      
  • Figure to Illustrate the Iterative Classification Process
    • 
-      
  • AdaBoost Examples
    • 
-      
  • Gradient boosting: Basics
    • 
-      
  • Gradient Boosting, algorithm
    • 
-      
  • Gradient Boosting, Examples
    • 
-      
  • Gradient Boots with Early Stopping
    • 
-      
  • XGBoost: Extreme Gradient Boosting
    • 
-      
  • Xgboost on the Cancer Data
    • 
+      
  • Making our own Bagging with Bootstrap
    • 
+      
  • Random forests
    • 
+      
  • Random Forest Algorithm
    • 
+      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
+      
  • Compare  Bagging on Trees with Random Forests
    • 
+      
  • Bootstrap with Random Forests Instead of a Single Tree
    • 
+      
  • Boosting, a Bird'e Eye
    • 
+      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
+      
  • Basic Steps of AdaBoost
    • 
+      
  • Figure to Illustrate the Iterative Classification Process
    • 
+      
  • AdaBoost Examples
    • 
+      
  • Gradient boosting: Basics
    • 
+      
  • Gradient Boosting, algorithm
    • 
+      
  • Gradient Boosting, Examples
    • 
+      
  • Gradient Boots with Early Stopping
    • 
+      
  • XGBoost: Extreme Gradient Boosting
    • 
+      
  • Xgboost on the Cancer Data
    • 
 
         
       
@@ -234,11 +240,25 @@ MathJax.Hub.Config({
 
 
 
-
    Feature Importance 
    
-
+
    Compare  Bagging on Trees with Random Forests  
    
 
    
-Example will be added here.
 
+
+
    bag_clf = BaggingClassifier(
+    DecisionTreeClassifier(splitter="random", max_leaf_nodes=16, random_state=42),
+    n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1, random_state=42)
+
    
+
    
+
+
+
    bag_clf.fit(X_train, y_train)
+y_pred = bag_clf.predict(X_test)
+from sklearn.ensemble import RandomForestClassifier
+rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)
+rnd_clf.fit(X_train, y_train)
+y_pred_rf = rnd_clf.predict(X_test)
+np.sum(y_pred == y_pred_rf) / len(y_pred) 
+
    
 
    
 
    
 
@@ -265,7 +285,7 @@ Example will be added here.
   
  • 53
    • 
   
  • 54
    • 
   
  • ...
    • 
-  
  • 56
    • 
+  
  • 57
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs045.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs045.html
index 94e51ba14..1431d4eb5 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs045.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs045.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
               ('Final regressor code', 2, None, '___sec28'),
               ('Pros and cons of trees, pros', 2, None, '___sec29'),
               ('Disadvantages', 2, None, '___sec30'),
-              ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+              ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+               'Boosting, Meet the Jungle of Methods',
                2,
                None,
                '___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
                None,
                '___sec37'),
               ('Now Bagging', 2, None, '___sec38'),
-              ('Random forests', 2, None, '___sec39'),
-              ('Random Forest Algorithm', 2, None, '___sec40'),
+              ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+              ('Random forests', 2, None, '___sec40'),
+              ('Random Forest Algorithm', 2, None, '___sec41'),
               ('Random Forests Compared with other Methods on the Cancer Data',
                2,
                None,
-               '___sec41'),
+               '___sec42'),
               ('Compare  Bagging on Trees with Random Forests',
                2,
                None,
-               '___sec42'),
-              ('Feature Importance', 2, None, '___sec43'),
-              ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+               '___sec43'),
+              ('Bootstrap with Random Forests Instead of a Single Tree',
+               2,
+               None,
+               '___sec44'),
+              ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
               ('Adaptive boosting: AdaBoost, Basic Algorithm',
                2,
                None,
-               '___sec45'),
-              ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+               '___sec46'),
+              ('Basic Steps of AdaBoost', 2, None, '___sec47'),
               ('Figure to Illustrate the Iterative Classification Process',
                2,
                None,
-               '___sec47'),
-              ('AdaBoost Examples', 2, None, '___sec48'),
-              ('Gradient boosting: Basics', 2, None, '___sec49'),
-              ('Gradient Boosting, algorithm', 2, None, '___sec50'),
-              ('Gradient Boosting, Examples', 2, None, '___sec51'),
-              ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
-              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
-              ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+               '___sec48'),
+              ('AdaBoost Examples', 2, None, '___sec49'),
+              ('Gradient boosting: Basics', 2, None, '___sec50'),
+              ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+              ('Gradient Boosting, Examples', 2, None, '___sec52'),
+              ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+              ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
 end of tocinfo -->
 
 
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
       
  • Final regressor code
    • 
       
  • Pros and cons of trees, pros
    • 
       
  • Disadvantages
    • 
-      
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • 
+      
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
    • 
       
  • An Overview of Ensemble Methods
    • 
       
  • Bagging
    • 
       
  • More bagging
    • 
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
       
  • Using the Voting Classifier
    • 
       
  • Please, not the moons again! Voting and Bagging
    • 
       
  • Now Bagging
    • 
-      
  • Random forests
    • 
-      
  • Random Forest Algorithm
    • 
-      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
-      
  • Compare  Bagging on Trees with Random Forests
    • 
-      
  • Feature Importance
    • 
-      
  • Boosting, a Bird'e Eye
    • 
-      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
-      
  • Basic Steps of AdaBoost
    • 
-      
  • Figure to Illustrate the Iterative Classification Process
    • 
-      
  • AdaBoost Examples
    • 
-      
  • Gradient boosting: Basics
    • 
-      
  • Gradient Boosting, algorithm
    • 
-      
  • Gradient Boosting, Examples
    • 
-      
  • Gradient Boots with Early Stopping
    • 
-      
  • XGBoost: Extreme Gradient Boosting
    • 
-      
  • Xgboost on the Cancer Data
    • 
+      
  • Making our own Bagging with Bootstrap
    • 
+      
  • Random forests
    • 
+      
  • Random Forest Algorithm
    • 
+      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
+      
  • Compare  Bagging on Trees with Random Forests
    • 
+      
  • Bootstrap with Random Forests Instead of a Single Tree
    • 
+      
  • Boosting, a Bird'e Eye
    • 
+      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
+      
  • Basic Steps of AdaBoost
    • 
+      
  • Figure to Illustrate the Iterative Classification Process
    • 
+      
  • AdaBoost Examples
    • 
+      
  • Gradient boosting: Basics
    • 
+      
  • Gradient Boosting, algorithm
    • 
+      
  • Gradient Boosting, Examples
    • 
+      
  • Gradient Boots with Early Stopping
    • 
+      
  • XGBoost: Extreme Gradient Boosting
    • 
+      
  • Xgboost on the Cancer Data
    • 
 
         
       
@@ -234,20 +240,63 @@ MathJax.Hub.Config({
 
 
 
-
    Boosting, a Bird'e Eye 
    
+
    Bootstrap with Random Forests Instead of a Single Tree 
    
 
 
    
-The basic idea is to combine weak classifiers in order to create a good
-classifier. With a weak classifier we often intend a classifier which
-produces results which are only slightly better than we would get by
-random guesses.
 
-
    
-This is done by applying in an iterative way a weak (or a standard
-classifier like decision trees) to modify the data. In each iteration
-we emphasize those observations which are misclassified by weighting
-them with a factor.
+
+
    import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.ensemble import RandomForestRegressor
 
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+    model = RandomForestRegressor()
+    y_pred = np.empty((y_test.shape[0], n_boostraps))
+    for i in range(n_boostraps):
+        x_, y_ = resample(X_train_scaled, y_train)
+        model.fit(x_, y_)
+        y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+    polydegree[degree] = degree
+    error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+    bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+    variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+    print('Polynomial degree:', degree)
+    print('Error:', error[degree])
+    print('Bias^2:', bias[degree])
+    print('Var:', variance[degree])
+    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
    
 
    
 
    
 
@@ -274,7 +323,7 @@ them with a factor.
   
  • 54
    • 
   
  • 55
    • 
   
  • ...
    • 
-  
  • 56
    • 
+  
  • 57
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs046.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs046.html
index 1eccd68a0..78f3f81a4 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs046.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs046.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
               ('Final regressor code', 2, None, '___sec28'),
               ('Pros and cons of trees, pros', 2, None, '___sec29'),
               ('Disadvantages', 2, None, '___sec30'),
-              ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+              ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+               'Boosting, Meet the Jungle of Methods',
                2,
                None,
                '___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
                None,
                '___sec37'),
               ('Now Bagging', 2, None, '___sec38'),
-              ('Random forests', 2, None, '___sec39'),
-              ('Random Forest Algorithm', 2, None, '___sec40'),
+              ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+              ('Random forests', 2, None, '___sec40'),
+              ('Random Forest Algorithm', 2, None, '___sec41'),
               ('Random Forests Compared with other Methods on the Cancer Data',
                2,
                None,
-               '___sec41'),
+               '___sec42'),
               ('Compare  Bagging on Trees with Random Forests',
                2,
                None,
-               '___sec42'),
-              ('Feature Importance', 2, None, '___sec43'),
-              ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+               '___sec43'),
+              ('Bootstrap with Random Forests Instead of a Single Tree',
+               2,
+               None,
+               '___sec44'),
+              ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
               ('Adaptive boosting: AdaBoost, Basic Algorithm',
                2,
                None,
-               '___sec45'),
-              ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+               '___sec46'),
+              ('Basic Steps of AdaBoost', 2, None, '___sec47'),
               ('Figure to Illustrate the Iterative Classification Process',
                2,
                None,
-               '___sec47'),
-              ('AdaBoost Examples', 2, None, '___sec48'),
-              ('Gradient boosting: Basics', 2, None, '___sec49'),
-              ('Gradient Boosting, algorithm', 2, None, '___sec50'),
-              ('Gradient Boosting, Examples', 2, None, '___sec51'),
-              ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
-              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
-              ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+               '___sec48'),
+              ('AdaBoost Examples', 2, None, '___sec49'),
+              ('Gradient boosting: Basics', 2, None, '___sec50'),
+              ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+              ('Gradient Boosting, Examples', 2, None, '___sec52'),
+              ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+              ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
 end of tocinfo -->
 
 
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
       
  • Final regressor code
    • 
       
  • Pros and cons of trees, pros
    • 
       
  • Disadvantages
    • 
-      
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • 
+      
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
    • 
       
  • An Overview of Ensemble Methods
    • 
       
  • Bagging
    • 
       
  • More bagging
    • 
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
       
  • Using the Voting Classifier
    • 
       
  • Please, not the moons again! Voting and Bagging
    • 
       
  • Now Bagging
    • 
-      
  • Random forests
    • 
-      
  • Random Forest Algorithm
    • 
-      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
-      
  • Compare  Bagging on Trees with Random Forests
    • 
-      
  • Feature Importance
    • 
-      
  • Boosting, a Bird'e Eye
    • 
-      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
-      
  • Basic Steps of AdaBoost
    • 
-      
  • Figure to Illustrate the Iterative Classification Process
    • 
-      
  • AdaBoost Examples
    • 
-      
  • Gradient boosting: Basics
    • 
-      
  • Gradient Boosting, algorithm
    • 
-      
  • Gradient Boosting, Examples
    • 
-      
  • Gradient Boots with Early Stopping
    • 
-      
  • XGBoost: Extreme Gradient Boosting
    • 
-      
  • Xgboost on the Cancer Data
    • 
+      
  • Making our own Bagging with Bootstrap
    • 
+      
  • Random forests
    • 
+      
  • Random Forest Algorithm
    • 
+      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
+      
  • Compare  Bagging on Trees with Random Forests
    • 
+      
  • Bootstrap with Random Forests Instead of a Single Tree
    • 
+      
  • Boosting, a Bird'e Eye
    • 
+      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
+      
  • Basic Steps of AdaBoost
    • 
+      
  • Figure to Illustrate the Iterative Classification Process
    • 
+      
  • AdaBoost Examples
    • 
+      
  • Gradient boosting: Basics
    • 
+      
  • Gradient Boosting, algorithm
    • 
+      
  • Gradient Boosting, Examples
    • 
+      
  • Gradient Boots with Early Stopping
    • 
+      
  • XGBoost: Extreme Gradient Boosting
    • 
+      
  • Xgboost on the Cancer Data
    • 
 
         
       
@@ -234,24 +240,19 @@ MathJax.Hub.Config({
 
 
 
-
    Adaptive boosting: AdaBoost, Basic Algorithm 
    
+
    Boosting, a Bird'e Eye 
    
 
 
    
-The algorithm here is rather straightforward. Assume that our weak
-classifier is a decision tree and we consider a binary set of outputs
-with \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
-observations. Our design matrix is given in terms of the
-feature/predictor vectors
-\( \boldsymbol{X}=[\boldsymbol{x}_0\boldsymbol{x}_1\dots\boldsymbol{x}_{p-1} \). Finally, we define also a
-classifier determined by our data via a function \( G(\boldsymbol{X}) \). This function tells us how well we are able to classify our outputs/targets \( \boldsymbol{y} \).
+The basic idea is to combine weak classifiers in order to create a good
+classifier. With a weak classifier we often intend a classifier which
+produces results which are only slightly better than we would get by
+random guesses.
 
 
    
-We can then define the misclassification error \( \mathrm{err} \) as
-$$
-\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(\boldsymbol{X}_{i*}),
-$$
-
-where the function \( I() \) is one if we misclassify and zero if we classify correctly.
+This is done by applying in an iterative way a weak (or a standard
+classifier like decision trees) to modify the data. In each iteration
+we emphasize those observations which are misclassified by weighting
+them with a factor.
 
 
    
 
    
@@ -278,6 +279,8 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
   
  • 54
    • 
   
  • 55
    • 
   
  • 56
    • 
+  
  • ...
    • 
+  
  • 57
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs047.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs047.html
index 35680be78..6cddc74f8 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs047.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs047.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
               ('Final regressor code', 2, None, '___sec28'),
               ('Pros and cons of trees, pros', 2, None, '___sec29'),
               ('Disadvantages', 2, None, '___sec30'),
-              ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+              ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+               'Boosting, Meet the Jungle of Methods',
                2,
                None,
                '___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
                None,
                '___sec37'),
               ('Now Bagging', 2, None, '___sec38'),
-              ('Random forests', 2, None, '___sec39'),
-              ('Random Forest Algorithm', 2, None, '___sec40'),
+              ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+              ('Random forests', 2, None, '___sec40'),
+              ('Random Forest Algorithm', 2, None, '___sec41'),
               ('Random Forests Compared with other Methods on the Cancer Data',
                2,
                None,
-               '___sec41'),
+               '___sec42'),
               ('Compare  Bagging on Trees with Random Forests',
                2,
                None,
-               '___sec42'),
-              ('Feature Importance', 2, None, '___sec43'),
-              ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+               '___sec43'),
+              ('Bootstrap with Random Forests Instead of a Single Tree',
+               2,
+               None,
+               '___sec44'),
+              ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
               ('Adaptive boosting: AdaBoost, Basic Algorithm',
                2,
                None,
-               '___sec45'),
-              ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+               '___sec46'),
+              ('Basic Steps of AdaBoost', 2, None, '___sec47'),
               ('Figure to Illustrate the Iterative Classification Process',
                2,
                None,
-               '___sec47'),
-              ('AdaBoost Examples', 2, None, '___sec48'),
-              ('Gradient boosting: Basics', 2, None, '___sec49'),
-              ('Gradient Boosting, algorithm', 2, None, '___sec50'),
-              ('Gradient Boosting, Examples', 2, None, '___sec51'),
-              ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
-              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
-              ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+               '___sec48'),
+              ('AdaBoost Examples', 2, None, '___sec49'),
+              ('Gradient boosting: Basics', 2, None, '___sec50'),
+              ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+              ('Gradient Boosting, Examples', 2, None, '___sec52'),
+              ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+              ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
 end of tocinfo -->
 
 
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
       
  • Final regressor code
    • 
       
  • Pros and cons of trees, pros
    • 
       
  • Disadvantages
    • 
-      
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • 
+      
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
    • 
       
  • An Overview of Ensemble Methods
    • 
       
  • Bagging
    • 
       
  • More bagging
    • 
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
       
  • Using the Voting Classifier
    • 
       
  • Please, not the moons again! Voting and Bagging
    • 
       
  • Now Bagging
    • 
-      
  • Random forests
    • 
-      
  • Random Forest Algorithm
    • 
-      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
-      
  • Compare  Bagging on Trees with Random Forests
    • 
-      
  • Feature Importance
    • 
-      
  • Boosting, a Bird'e Eye
    • 
-      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
-      
  • Basic Steps of AdaBoost
    • 
-      
  • Figure to Illustrate the Iterative Classification Process
    • 
-      
  • AdaBoost Examples
    • 
-      
  • Gradient boosting: Basics
    • 
-      
  • Gradient Boosting, algorithm
    • 
-      
  • Gradient Boosting, Examples
    • 
-      
  • Gradient Boots with Early Stopping
    • 
-      
  • XGBoost: Extreme Gradient Boosting
    • 
-      
  • Xgboost on the Cancer Data
    • 
+      
  • Making our own Bagging with Bootstrap
    • 
+      
  • Random forests
    • 
+      
  • Random Forest Algorithm
    • 
+      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
+      
  • Compare  Bagging on Trees with Random Forests
    • 
+      
  • Bootstrap with Random Forests Instead of a Single Tree
    • 
+      
  • Boosting, a Bird'e Eye
    • 
+      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
+      
  • Basic Steps of AdaBoost
    • 
+      
  • Figure to Illustrate the Iterative Classification Process
    • 
+      
  • AdaBoost Examples
    • 
+      
  • Gradient boosting: Basics
    • 
+      
  • Gradient Boosting, algorithm
    • 
+      
  • Gradient Boosting, Examples
    • 
+      
  • Gradient Boots with Early Stopping
    • 
+      
  • XGBoost: Extreme Gradient Boosting
    • 
+      
  • Xgboost on the Cancer Data
    • 
 
         
       
@@ -234,42 +240,24 @@ MathJax.Hub.Config({
 
 
 
-
    Basic Steps of AdaBoost 
    
+
    Adaptive boosting: AdaBoost, Basic Algorithm 
    
 
 
    
-With the above definitions we are now ready to set up the algorithm for AdaBoost.
-The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases.
-
-
      
-
    1.  We start by initializing all weights to \( w_i = 1/n \), with \( i=0,1,2,\dots n-1 \). It is to see then that \( \sum_{i=0}^{n-1}w_i = 1 \).
      2. 
-
    2.  We rewrite the misclassification error as
      3.  
-
    
+The algorithm here is rather straightforward. Assume that our weak
+classifier is a decision tree and we consider a binary set of outputs
+with \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
+observations. Our design matrix is given in terms of the
+feature/predictor vectors
+\( \boldsymbol{X}=[\boldsymbol{x}_0\boldsymbol{x}_1\dots\boldsymbol{x}_{p-1} \). Finally, we define also a
+classifier determined by our data via a function \( G(\boldsymbol{X}) \). This function tells us how well we are able to classify our outputs/targets \( \boldsymbol{y} \).
 
+
    
+We can then define the misclassification error \( \mathrm{err} \) as
 $$
-\mathrm{err}=\frac{\sum_{i=0}^{n-1}w_iI(y_i\ne G(\boldsymbol{X}_{i*})}{\sum_{i=0}^{n-1}w_i},
+\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(\boldsymbol{X}_{i*}),
 $$
 
-
-
      
-
    1.  Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree.
-
-
        
- 
      1.  Fit thus a given classifier to the training using the weights \( w_i \).
        2. 
- 
      2.  Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.
        3. 
- 
      3.  Define a quantity $\alpha_{m} = \log{(1-\mathrm{err})/\mathrm{err}}
        4. 
- 
      4.  Set the new weights to $w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(\boldsymbol{X}_{i*})}.
        5. 
-
      
-
-
    2.  Compute the new classifier $G(\boldsymbol{X})= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(\boldsymbol{X}_{i*}).
      3. 
-
    
-
-For the iterations with \( m \le 2 \) the weights are modified
-individually at each steps. The obersvations which were misclassified
-at iteration \( m-1 \) have a weight which is larger than those which were
-classified properly. As this proceeds, the observations which were
-difficult to classifiy correctly are given a larger influence. Each
-new classificatio step \( m \) is then forced to concentrate on those
-observations that are missed in the previous iterations.
+where the function \( I() \) is one if we misclassify and zero if we classify correctly.
 
 
    
 
    
@@ -295,6 +283,7 @@ observations that are missed in the previous iterations.
   
  • 54
    • 
   
  • 55
    • 
   
  • 56
    • 
+  
  • 57
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs048.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs048.html
index 32e455b1e..e13cc85c8 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs048.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs048.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
               ('Final regressor code', 2, None, '___sec28'),
               ('Pros and cons of trees, pros', 2, None, '___sec29'),
               ('Disadvantages', 2, None, '___sec30'),
-              ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+              ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+               'Boosting, Meet the Jungle of Methods',
                2,
                None,
                '___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
                None,
                '___sec37'),
               ('Now Bagging', 2, None, '___sec38'),
-              ('Random forests', 2, None, '___sec39'),
-              ('Random Forest Algorithm', 2, None, '___sec40'),
+              ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+              ('Random forests', 2, None, '___sec40'),
+              ('Random Forest Algorithm', 2, None, '___sec41'),
               ('Random Forests Compared with other Methods on the Cancer Data',
                2,
                None,
-               '___sec41'),
+               '___sec42'),
               ('Compare  Bagging on Trees with Random Forests',
                2,
                None,
-               '___sec42'),
-              ('Feature Importance', 2, None, '___sec43'),
-              ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+               '___sec43'),
+              ('Bootstrap with Random Forests Instead of a Single Tree',
+               2,
+               None,
+               '___sec44'),
+              ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
               ('Adaptive boosting: AdaBoost, Basic Algorithm',
                2,
                None,
-               '___sec45'),
-              ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+               '___sec46'),
+              ('Basic Steps of AdaBoost', 2, None, '___sec47'),
               ('Figure to Illustrate the Iterative Classification Process',
                2,
                None,
-               '___sec47'),
-              ('AdaBoost Examples', 2, None, '___sec48'),
-              ('Gradient boosting: Basics', 2, None, '___sec49'),
-              ('Gradient Boosting, algorithm', 2, None, '___sec50'),
-              ('Gradient Boosting, Examples', 2, None, '___sec51'),
-              ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
-              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
-              ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+               '___sec48'),
+              ('AdaBoost Examples', 2, None, '___sec49'),
+              ('Gradient boosting: Basics', 2, None, '___sec50'),
+              ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+              ('Gradient Boosting, Examples', 2, None, '___sec52'),
+              ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+              ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
 end of tocinfo -->
 
 
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
       
  • Final regressor code
    • 
       
  • Pros and cons of trees, pros
    • 
       
  • Disadvantages
    • 
-      
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • 
+      
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
    • 
       
  • An Overview of Ensemble Methods
    • 
       
  • Bagging
    • 
       
  • More bagging
    • 
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
       
  • Using the Voting Classifier
    • 
       
  • Please, not the moons again! Voting and Bagging
    • 
       
  • Now Bagging
    • 
-      
  • Random forests
    • 
-      
  • Random Forest Algorithm
    • 
-      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
-      
  • Compare  Bagging on Trees with Random Forests
    • 
-      
  • Feature Importance
    • 
-      
  • Boosting, a Bird'e Eye
    • 
-      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
-      
  • Basic Steps of AdaBoost
    • 
-      
  • Figure to Illustrate the Iterative Classification Process
    • 
-      
  • AdaBoost Examples
    • 
-      
  • Gradient boosting: Basics
    • 
-      
  • Gradient Boosting, algorithm
    • 
-      
  • Gradient Boosting, Examples
    • 
-      
  • Gradient Boots with Early Stopping
    • 
-      
  • XGBoost: Extreme Gradient Boosting
    • 
-      
  • Xgboost on the Cancer Data
    • 
+      
  • Making our own Bagging with Bootstrap
    • 
+      
  • Random forests
    • 
+      
  • Random Forest Algorithm
    • 
+      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
+      
  • Compare  Bagging on Trees with Random Forests
    • 
+      
  • Bootstrap with Random Forests Instead of a Single Tree
    • 
+      
  • Boosting, a Bird'e Eye
    • 
+      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
+      
  • Basic Steps of AdaBoost
    • 
+      
  • Figure to Illustrate the Iterative Classification Process
    • 
+      
  • AdaBoost Examples
    • 
+      
  • Gradient boosting: Basics
    • 
+      
  • Gradient Boosting, algorithm
    • 
+      
  • Gradient Boosting, Examples
    • 
+      
  • Gradient Boots with Early Stopping
    • 
+      
  • XGBoost: Extreme Gradient Boosting
    • 
+      
  • Xgboost on the Cancer Data
    • 
 
         
       
@@ -234,7 +240,42 @@ MathJax.Hub.Config({
 
 
 
-
    Figure to Illustrate the Iterative Classification Process 
    
+
    Basic Steps of AdaBoost 
    
+
+
    
+With the above definitions we are now ready to set up the algorithm for AdaBoost.
+The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases.
+
+
      
+
    1.  We start by initializing all weights to \( w_i = 1/n \), with \( i=0,1,2,\dots n-1 \). It is to see then that \( \sum_{i=0}^{n-1}w_i = 1 \).
      2. 
+
    2.  We rewrite the misclassification error as
      3.  
+
    
+
+$$
+\mathrm{err}=\frac{\sum_{i=0}^{n-1}w_iI(y_i\ne G(\boldsymbol{X}_{i*})}{\sum_{i=0}^{n-1}w_i},
+$$
+
+
+
      
+
    1.  Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree.
+
+
        
+ 
      1.  Fit thus a given classifier to the training using the weights \( w_i \).
        2. 
+ 
      2.  Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.
        3. 
+ 
      3.  Define a quantity $\alpha_{m} = \log{(1-\mathrm{err})/\mathrm{err}}
        4. 
+ 
      4.  Set the new weights to $w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(\boldsymbol{X}_{i*})}.
        5. 
+
      
+
+
    2.  Compute the new classifier $G(\boldsymbol{X})= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(\boldsymbol{X}_{i*}).
      3. 
+
    
+
+For the iterations with \( m \le 2 \) the weights are modified
+individually at each steps. The obersvations which were misclassified
+at iteration \( m-1 \) have a weight which is larger than those which were
+classified properly. As this proceeds, the observations which were
+difficult to classifiy correctly are given a larger influence. Each
+new classification step \( m \) is then forced to concentrate on those
+observations that are missed in the previous iterations.
 
 
    
 
    
@@ -259,6 +300,7 @@ MathJax.Hub.Config({
   
  • 54
    • 
   
  • 55
    • 
   
  • 56
    • 
+  
  • 57
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs049.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs049.html
index f0981082a..4a6ad8a24 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs049.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs049.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
               ('Final regressor code', 2, None, '___sec28'),
               ('Pros and cons of trees, pros', 2, None, '___sec29'),
               ('Disadvantages', 2, None, '___sec30'),
-              ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+              ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+               'Boosting, Meet the Jungle of Methods',
                2,
                None,
                '___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
                None,
                '___sec37'),
               ('Now Bagging', 2, None, '___sec38'),
-              ('Random forests', 2, None, '___sec39'),
-              ('Random Forest Algorithm', 2, None, '___sec40'),
+              ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+              ('Random forests', 2, None, '___sec40'),
+              ('Random Forest Algorithm', 2, None, '___sec41'),
               ('Random Forests Compared with other Methods on the Cancer Data',
                2,
                None,
-               '___sec41'),
+               '___sec42'),
               ('Compare  Bagging on Trees with Random Forests',
                2,
                None,
-               '___sec42'),
-              ('Feature Importance', 2, None, '___sec43'),
-              ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+               '___sec43'),
+              ('Bootstrap with Random Forests Instead of a Single Tree',
+               2,
+               None,
+               '___sec44'),
+              ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
               ('Adaptive boosting: AdaBoost, Basic Algorithm',
                2,
                None,
-               '___sec45'),
-              ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+               '___sec46'),
+              ('Basic Steps of AdaBoost', 2, None, '___sec47'),
               ('Figure to Illustrate the Iterative Classification Process',
                2,
                None,
-               '___sec47'),
-              ('AdaBoost Examples', 2, None, '___sec48'),
-              ('Gradient boosting: Basics', 2, None, '___sec49'),
-              ('Gradient Boosting, algorithm', 2, None, '___sec50'),
-              ('Gradient Boosting, Examples', 2, None, '___sec51'),
-              ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
-              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
-              ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+               '___sec48'),
+              ('AdaBoost Examples', 2, None, '___sec49'),
+              ('Gradient boosting: Basics', 2, None, '___sec50'),
+              ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+              ('Gradient Boosting, Examples', 2, None, '___sec52'),
+              ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+              ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
 end of tocinfo -->
 
 
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
       
  • Final regressor code
    • 
       
  • Pros and cons of trees, pros
    • 
       
  • Disadvantages
    • 
-      
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • 
+      
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
    • 
       
  • An Overview of Ensemble Methods
    • 
       
  • Bagging
    • 
       
  • More bagging
    • 
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
       
  • Using the Voting Classifier
    • 
       
  • Please, not the moons again! Voting and Bagging
    • 
       
  • Now Bagging
    • 
-      
  • Random forests
    • 
-      
  • Random Forest Algorithm
    • 
-      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
-      
  • Compare  Bagging on Trees with Random Forests
    • 
-      
  • Feature Importance
    • 
-      
  • Boosting, a Bird'e Eye
    • 
-      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
-      
  • Basic Steps of AdaBoost
    • 
-      
  • Figure to Illustrate the Iterative Classification Process
    • 
-      
  • AdaBoost Examples
    • 
-      
  • Gradient boosting: Basics
    • 
-      
  • Gradient Boosting, algorithm
    • 
-      
  • Gradient Boosting, Examples
    • 
-      
  • Gradient Boots with Early Stopping
    • 
-      
  • XGBoost: Extreme Gradient Boosting
    • 
-      
  • Xgboost on the Cancer Data
    • 
+      
  • Making our own Bagging with Bootstrap
    • 
+      
  • Random forests
    • 
+      
  • Random Forest Algorithm
    • 
+      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
+      
  • Compare  Bagging on Trees with Random Forests
    • 
+      
  • Bootstrap with Random Forests Instead of a Single Tree
    • 
+      
  • Boosting, a Bird'e Eye
    • 
+      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
+      
  • Basic Steps of AdaBoost
    • 
+      
  • Figure to Illustrate the Iterative Classification Process
    • 
+      
  • AdaBoost Examples
    • 
+      
  • Gradient boosting: Basics
    • 
+      
  • Gradient Boosting, algorithm
    • 
+      
  • Gradient Boosting, Examples
    • 
+      
  • Gradient Boots with Early Stopping
    • 
+      
  • XGBoost: Extreme Gradient Boosting
    • 
+      
  • Xgboost on the Cancer Data
    • 
 
         
       
@@ -234,46 +240,8 @@ MathJax.Hub.Config({
 
 
 
-
    AdaBoost Examples 
    
+
    Figure to Illustrate the Iterative Classification Process 
    
 
-
    
-Using Scikit-Learn it is easy to appply the adaptive boosting algorithm, as done here.
-
-
    
-
-
-
    from sklearn.ensemble import AdaBoostClassifier
-
-ada_clf = AdaBoostClassifier(
-    DecisionTreeClassifier(max_depth=1), n_estimators=200,
-    algorithm="SAMME.R", learning_rate=0.5, random_state=42)
-ada_clf.fit(X_train, y_train)
-
-plot_decision_boundary(ada_clf, X, y)
-
-m = len(X_train)
-
-plt.figure(figsize=(11, 4))
-for subplot, learning_rate in ((121, 1), (122, 0.5)):
-    sample_weights = np.ones(m)
-    plt.subplot(subplot)
-    for i in range(5):
-        svm_clf = SVC(kernel="rbf", C=0.05, gamma="auto", random_state=42)
-        svm_clf.fit(X_train, y_train, sample_weight=sample_weights)
-        y_pred = svm_clf.predict(X_train)
-        sample_weights[y_pred != y_train] *= (1 + learning_rate)
-        plot_decision_boundary(svm_clf, X, y, alpha=0.2)
-        plt.title("learning_rate = {}".format(learning_rate), fontsize=16)
-    if subplot == 121:
-        plt.text(-0.7, -0.65, "1", fontsize=14)
-        plt.text(-0.6, -0.10, "2", fontsize=14)
-        plt.text(-0.5,  0.10, "3", fontsize=14)
-        plt.text(-0.4,  0.55, "4", fontsize=14)
-        plt.text(-0.3,  0.90, "5", fontsize=14)
-
-save_fig("boosting_plot")
-plt.show()
-
    
 
    
 
    
 
@@ -296,6 +264,7 @@ plt.show()
   
  • 54
    • 
   
  • 55
    • 
   
  • 56
    • 
+  
  • 57
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs050.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs050.html
index 987ba0b72..4d7d6c3fa 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs050.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs050.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
               ('Final regressor code', 2, None, '___sec28'),
               ('Pros and cons of trees, pros', 2, None, '___sec29'),
               ('Disadvantages', 2, None, '___sec30'),
-              ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+              ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+               'Boosting, Meet the Jungle of Methods',
                2,
                None,
                '___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
                None,
                '___sec37'),
               ('Now Bagging', 2, None, '___sec38'),
-              ('Random forests', 2, None, '___sec39'),
-              ('Random Forest Algorithm', 2, None, '___sec40'),
+              ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+              ('Random forests', 2, None, '___sec40'),
+              ('Random Forest Algorithm', 2, None, '___sec41'),
               ('Random Forests Compared with other Methods on the Cancer Data',
                2,
                None,
-               '___sec41'),
+               '___sec42'),
               ('Compare  Bagging on Trees with Random Forests',
                2,
                None,
-               '___sec42'),
-              ('Feature Importance', 2, None, '___sec43'),
-              ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+               '___sec43'),
+              ('Bootstrap with Random Forests Instead of a Single Tree',
+               2,
+               None,
+               '___sec44'),
+              ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
               ('Adaptive boosting: AdaBoost, Basic Algorithm',
                2,
                None,
-               '___sec45'),
-              ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+               '___sec46'),
+              ('Basic Steps of AdaBoost', 2, None, '___sec47'),
               ('Figure to Illustrate the Iterative Classification Process',
                2,
                None,
-               '___sec47'),
-              ('AdaBoost Examples', 2, None, '___sec48'),
-              ('Gradient boosting: Basics', 2, None, '___sec49'),
-              ('Gradient Boosting, algorithm', 2, None, '___sec50'),
-              ('Gradient Boosting, Examples', 2, None, '___sec51'),
-              ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
-              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
-              ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+               '___sec48'),
+              ('AdaBoost Examples', 2, None, '___sec49'),
+              ('Gradient boosting: Basics', 2, None, '___sec50'),
+              ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+              ('Gradient Boosting, Examples', 2, None, '___sec52'),
+              ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+              ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
 end of tocinfo -->
 
 
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
       
  • Final regressor code
    • 
       
  • Pros and cons of trees, pros
    • 
       
  • Disadvantages
    • 
-      
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • 
+      
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
    • 
       
  • An Overview of Ensemble Methods
    • 
       
  • Bagging
    • 
       
  • More bagging
    • 
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
       
  • Using the Voting Classifier
    • 
       
  • Please, not the moons again! Voting and Bagging
    • 
       
  • Now Bagging
    • 
-      
  • Random forests
    • 
-      
  • Random Forest Algorithm
    • 
-      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
-      
  • Compare  Bagging on Trees with Random Forests
    • 
-      
  • Feature Importance
    • 
-      
  • Boosting, a Bird'e Eye
    • 
-      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
-      
  • Basic Steps of AdaBoost
    • 
-      
  • Figure to Illustrate the Iterative Classification Process
    • 
-      
  • AdaBoost Examples
    • 
-      
  • Gradient boosting: Basics
    • 
-      
  • Gradient Boosting, algorithm
    • 
-      
  • Gradient Boosting, Examples
    • 
-      
  • Gradient Boots with Early Stopping
    • 
-      
  • XGBoost: Extreme Gradient Boosting
    • 
-      
  • Xgboost on the Cancer Data
    • 
+      
  • Making our own Bagging with Bootstrap
    • 
+      
  • Random forests
    • 
+      
  • Random Forest Algorithm
    • 
+      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
+      
  • Compare  Bagging on Trees with Random Forests
    • 
+      
  • Bootstrap with Random Forests Instead of a Single Tree
    • 
+      
  • Boosting, a Bird'e Eye
    • 
+      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
+      
  • Basic Steps of AdaBoost
    • 
+      
  • Figure to Illustrate the Iterative Classification Process
    • 
+      
  • AdaBoost Examples
    • 
+      
  • Gradient boosting: Basics
    • 
+      
  • Gradient Boosting, algorithm
    • 
+      
  • Gradient Boosting, Examples
    • 
+      
  • Gradient Boots with Early Stopping
    • 
+      
  • XGBoost: Extreme Gradient Boosting
    • 
+      
  • Xgboost on the Cancer Data
    • 
 
         
       
@@ -234,18 +240,46 @@ MathJax.Hub.Config({
 
 
 
-
    Gradient boosting: Basics 
    
+
    AdaBoost Examples 
    
 
 
    
-Gradient boosting is again a similar technique to Adapative boosting,
-it combines so-called weak classifiers or regressors into a strong
-method via a series of iterations.
+Using Scikit-Learn it is easy to appply the adaptive boosting algorithm, as done here.
 
 
    
-In order to understand the method, let us illustrate its basics by
-bringing back the essential steps in linear regression, where our cost
-function was the least squares function.
 
+
+
    from sklearn.ensemble import AdaBoostClassifier
+
+ada_clf = AdaBoostClassifier(
+    DecisionTreeClassifier(max_depth=1), n_estimators=200,
+    algorithm="SAMME.R", learning_rate=0.5, random_state=42)
+ada_clf.fit(X_train, y_train)
+
+plot_decision_boundary(ada_clf, X, y)
+
+m = len(X_train)
+
+plt.figure(figsize=(11, 4))
+for subplot, learning_rate in ((121, 1), (122, 0.5)):
+    sample_weights = np.ones(m)
+    plt.subplot(subplot)
+    for i in range(5):
+        svm_clf = SVC(kernel="rbf", C=0.05, gamma="auto", random_state=42)
+        svm_clf.fit(X_train, y_train, sample_weight=sample_weights)
+        y_pred = svm_clf.predict(X_train)
+        sample_weights[y_pred != y_train] *= (1 + learning_rate)
+        plot_decision_boundary(svm_clf, X, y, alpha=0.2)
+        plt.title("learning_rate = {}".format(learning_rate), fontsize=16)
+    if subplot == 121:
+        plt.text(-0.7, -0.65, "1", fontsize=14)
+        plt.text(-0.6, -0.10, "2", fontsize=14)
+        plt.text(-0.5,  0.10, "3", fontsize=14)
+        plt.text(-0.4,  0.55, "4", fontsize=14)
+        plt.text(-0.3,  0.90, "5", fontsize=14)
+
+save_fig("boosting_plot")
+plt.show()
+
    
 
    
 
    
 
@@ -267,6 +301,7 @@ function was the least squares function.
   
  • 54
    • 
   
  • 55
    • 
   
  • 56
    • 
+  
  • 57
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs051.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs051.html
index 94873b2b4..fe4dd40f5 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs051.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs051.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
               ('Final regressor code', 2, None, '___sec28'),
               ('Pros and cons of trees, pros', 2, None, '___sec29'),
               ('Disadvantages', 2, None, '___sec30'),
-              ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+              ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+               'Boosting, Meet the Jungle of Methods',
                2,
                None,
                '___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
                None,
                '___sec37'),
               ('Now Bagging', 2, None, '___sec38'),
-              ('Random forests', 2, None, '___sec39'),
-              ('Random Forest Algorithm', 2, None, '___sec40'),
+              ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+              ('Random forests', 2, None, '___sec40'),
+              ('Random Forest Algorithm', 2, None, '___sec41'),
               ('Random Forests Compared with other Methods on the Cancer Data',
                2,
                None,
-               '___sec41'),
+               '___sec42'),
               ('Compare  Bagging on Trees with Random Forests',
                2,
                None,
-               '___sec42'),
-              ('Feature Importance', 2, None, '___sec43'),
-              ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+               '___sec43'),
+              ('Bootstrap with Random Forests Instead of a Single Tree',
+               2,
+               None,
+               '___sec44'),
+              ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
               ('Adaptive boosting: AdaBoost, Basic Algorithm',
                2,
                None,
-               '___sec45'),
-              ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+               '___sec46'),
+              ('Basic Steps of AdaBoost', 2, None, '___sec47'),
               ('Figure to Illustrate the Iterative Classification Process',
                2,
                None,
-               '___sec47'),
-              ('AdaBoost Examples', 2, None, '___sec48'),
-              ('Gradient boosting: Basics', 2, None, '___sec49'),
-              ('Gradient Boosting, algorithm', 2, None, '___sec50'),
-              ('Gradient Boosting, Examples', 2, None, '___sec51'),
-              ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
-              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
-              ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+               '___sec48'),
+              ('AdaBoost Examples', 2, None, '___sec49'),
+              ('Gradient boosting: Basics', 2, None, '___sec50'),
+              ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+              ('Gradient Boosting, Examples', 2, None, '___sec52'),
+              ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+              ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
 end of tocinfo -->
 
 
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
       
  • Final regressor code
    • 
       
  • Pros and cons of trees, pros
    • 
       
  • Disadvantages
    • 
-      
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • 
+      
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
    • 
       
  • An Overview of Ensemble Methods
    • 
       
  • Bagging
    • 
       
  • More bagging
    • 
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
       
  • Using the Voting Classifier
    • 
       
  • Please, not the moons again! Voting and Bagging
    • 
       
  • Now Bagging
    • 
-      
  • Random forests
    • 
-      
  • Random Forest Algorithm
    • 
-      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
-      
  • Compare  Bagging on Trees with Random Forests
    • 
-      
  • Feature Importance
    • 
-      
  • Boosting, a Bird'e Eye
    • 
-      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
-      
  • Basic Steps of AdaBoost
    • 
-      
  • Figure to Illustrate the Iterative Classification Process
    • 
-      
  • AdaBoost Examples
    • 
-      
  • Gradient boosting: Basics
    • 
-      
  • Gradient Boosting, algorithm
    • 
-      
  • Gradient Boosting, Examples
    • 
-      
  • Gradient Boots with Early Stopping
    • 
-      
  • XGBoost: Extreme Gradient Boosting
    • 
-      
  • Xgboost on the Cancer Data
    • 
+      
  • Making our own Bagging with Bootstrap
    • 
+      
  • Random forests
    • 
+      
  • Random Forest Algorithm
    • 
+      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
+      
  • Compare  Bagging on Trees with Random Forests
    • 
+      
  • Bootstrap with Random Forests Instead of a Single Tree
    • 
+      
  • Boosting, a Bird'e Eye
    • 
+      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
+      
  • Basic Steps of AdaBoost
    • 
+      
  • Figure to Illustrate the Iterative Classification Process
    • 
+      
  • AdaBoost Examples
    • 
+      
  • Gradient boosting: Basics
    • 
+      
  • Gradient Boosting, algorithm
    • 
+      
  • Gradient Boosting, Examples
    • 
+      
  • Gradient Boots with Early Stopping
    • 
+      
  • XGBoost: Extreme Gradient Boosting
    • 
+      
  • Xgboost on the Cancer Data
    • 
 
         
       
@@ -234,30 +240,19 @@ MathJax.Hub.Config({
 
 
 
-
    Gradient Boosting, algorithm 
    
+
    Gradient boosting: Basics 
    
 
 
    
-Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard least squares  function
-$$
-C(\boldsymbol{y},\boldsymbol{f})=\frac{1}{n}\sum_{i=0}^{n-1}(y_i-f(x_i))^2.
-$$
+Gradient boosting is again a similar technique to Adapative boosting,
+it combines so-called weak classifiers or regressors into a strong
+method via a series of iterations.
 
 
    
-The way we proceed in an iterative fashion is to
-
-
      
-
    1.  Initialize our estimate by \( f_0(x)=0 \).
      2. 
-
    2.  For \( m=1:M \), we
-
-
        
- 
      1.  compute the negative gradient vector \( \boldsymbol{u}_m = -\partial C(\boldsymbol{y},\boldsymbol{f})/\partial \boldsymbol{f}(x) \) at $f(x) = f_{m-1}(x);
        2. 
- 
      2.  fit the so-called base-learner to the negative gradient \( h_m(u_m,x) \);
        3. 
- 
      3.  update the estimate \( f_m(x) = f_{m-1}(x)+\nu h_m(u_m,x) \);
        4. 
-
      
-
-
    3.  The final estimate is then \( f_M(x) = \sum_{m=1}^M\nu h_m(u_m,x) \).
      4. 
-
    
+In order to understand the method, let us illustrate its basics by
+bringing back the essential steps in linear regression, where our cost
+function was the least squares function.
 
+
    
 
    
 
 
      
@@ -277,6 +272,7 @@ The way we proceed in an iterative fashion is to
   
    • 54
      • 
   
    • 55
      • 
   
    • 56
      • 
+  
    • 57
      • 
   
    • »
      • 
 
    
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs052.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs052.html
index a1d6b07ec..0f3729124 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs052.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs052.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
               ('Final regressor code', 2, None, '___sec28'),
               ('Pros and cons of trees, pros', 2, None, '___sec29'),
               ('Disadvantages', 2, None, '___sec30'),
-              ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+              ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+               'Boosting, Meet the Jungle of Methods',
                2,
                None,
                '___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
                None,
                '___sec37'),
               ('Now Bagging', 2, None, '___sec38'),
-              ('Random forests', 2, None, '___sec39'),
-              ('Random Forest Algorithm', 2, None, '___sec40'),
+              ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+              ('Random forests', 2, None, '___sec40'),
+              ('Random Forest Algorithm', 2, None, '___sec41'),
               ('Random Forests Compared with other Methods on the Cancer Data',
                2,
                None,
-               '___sec41'),
+               '___sec42'),
               ('Compare  Bagging on Trees with Random Forests',
                2,
                None,
-               '___sec42'),
-              ('Feature Importance', 2, None, '___sec43'),
-              ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+               '___sec43'),
+              ('Bootstrap with Random Forests Instead of a Single Tree',
+               2,
+               None,
+               '___sec44'),
+              ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
               ('Adaptive boosting: AdaBoost, Basic Algorithm',
                2,
                None,
-               '___sec45'),
-              ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+               '___sec46'),
+              ('Basic Steps of AdaBoost', 2, None, '___sec47'),
               ('Figure to Illustrate the Iterative Classification Process',
                2,
                None,
-               '___sec47'),
-              ('AdaBoost Examples', 2, None, '___sec48'),
-              ('Gradient boosting: Basics', 2, None, '___sec49'),
-              ('Gradient Boosting, algorithm', 2, None, '___sec50'),
-              ('Gradient Boosting, Examples', 2, None, '___sec51'),
-              ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
-              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
-              ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+               '___sec48'),
+              ('AdaBoost Examples', 2, None, '___sec49'),
+              ('Gradient boosting: Basics', 2, None, '___sec50'),
+              ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+              ('Gradient Boosting, Examples', 2, None, '___sec52'),
+              ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+              ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
 end of tocinfo -->
 
 
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
       
  • Final regressor code
    • 
       
  • Pros and cons of trees, pros
    • 
       
  • Disadvantages
    • 
-      
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • 
+      
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
    • 
       
  • An Overview of Ensemble Methods
    • 
       
  • Bagging
    • 
       
  • More bagging
    • 
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
       
  • Using the Voting Classifier
    • 
       
  • Please, not the moons again! Voting and Bagging
    • 
       
  • Now Bagging
    • 
-      
  • Random forests
    • 
-      
  • Random Forest Algorithm
    • 
-      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
-      
  • Compare  Bagging on Trees with Random Forests
    • 
-      
  • Feature Importance
    • 
-      
  • Boosting, a Bird'e Eye
    • 
-      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
-      
  • Basic Steps of AdaBoost
    • 
-      
  • Figure to Illustrate the Iterative Classification Process
    • 
-      
  • AdaBoost Examples
    • 
-      
  • Gradient boosting: Basics
    • 
-      
  • Gradient Boosting, algorithm
    • 
-      
  • Gradient Boosting, Examples
    • 
-      
  • Gradient Boots with Early Stopping
    • 
-      
  • XGBoost: Extreme Gradient Boosting
    • 
-      
  • Xgboost on the Cancer Data
    • 
+      
  • Making our own Bagging with Bootstrap
    • 
+      
  • Random forests
    • 
+      
  • Random Forest Algorithm
    • 
+      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
+      
  • Compare  Bagging on Trees with Random Forests
    • 
+      
  • Bootstrap with Random Forests Instead of a Single Tree
    • 
+      
  • Boosting, a Bird'e Eye
    • 
+      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
+      
  • Basic Steps of AdaBoost
    • 
+      
  • Figure to Illustrate the Iterative Classification Process
    • 
+      
  • AdaBoost Examples
    • 
+      
  • Gradient boosting: Basics
    • 
+      
  • Gradient Boosting, algorithm
    • 
+      
  • Gradient Boosting, Examples
    • 
+      
  • Gradient Boots with Early Stopping
    • 
+      
  • XGBoost: Extreme Gradient Boosting
    • 
+      
  • Xgboost on the Cancer Data
    • 
 
         
       
@@ -234,94 +240,30 @@ MathJax.Hub.Config({
 
 
 
-
    Gradient Boosting, Examples 
    
+
    Gradient Boosting, algorithm 
    
+
 
    
+Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard least squares  function
+$$
+C(\boldsymbol{y},\boldsymbol{f})=\frac{1}{n}\sum_{i=0}^{n-1}(y_i-f(x_i))^2.
+$$
 
-
-
    np.random.seed(42)
-X = np.random.rand(100, 1) - 0.5
-y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)
-
-from sklearn.tree import DecisionTreeRegressor
-
-tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg1.fit(X, y)
-
-y2 = y - tree_reg1.predict(X)
-tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg2.fit(X, y2)
-
-y3 = y2 - tree_reg2.predict(X)
-tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg3.fit(X, y3)
-
-X_new = np.array([[0.8]])
-y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))
-
-def plot_predictions(regressors, X, y, axes, label=None, style="r-", data_style="b.", data_label=None):
-    x1 = np.linspace(axes[0], axes[1], 500)
-    y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)
-    plt.plot(X[:, 0], y, data_style, label=data_label)
-    plt.plot(x1, y_pred, style, linewidth=2, label=label)
-    if label or data_label:
-        plt.legend(loc="upper center", fontsize=16)
-    plt.axis(axes)
-
-plt.figure(figsize=(11,11))
-
-plt.subplot(321)
-plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h_1(x_1)$", style="g-", data_label="Training set")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-plt.title("Residuals and tree predictions", fontsize=16)
-
-plt.subplot(322)
-plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1)$", data_label="Training set")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-plt.title("Ensemble predictions", fontsize=16)
-
-plt.subplot(323)
-plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_2(x_1)$", style="g-", data_style="k+", data_label="Residuals")
-plt.ylabel("$y - h_1(x_1)$", fontsize=16)
-
-plt.subplot(324)
-plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1)$")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-
-plt.subplot(325)
-plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_3(x_1)$", style="g-", data_style="k+")
-plt.ylabel("$y - h_1(x_1) - h_2(x_1)$", fontsize=16)
-plt.xlabel("$x_1$", fontsize=16)
-
-plt.subplot(326)
-plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$")
-plt.xlabel("$x_1$", fontsize=16)
-plt.ylabel("$y$", fontsize=16, rotation=0)
-
-save_fig("gradient_boosting_plot")
-plt.show()
-
-from sklearn.ensemble import GradientBoostingRegressor
-
-gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0, random_state=42)
-gbrt.fit(X, y)
-
-gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)
-gbrt_slow.fit(X, y)
-
-plt.figure(figsize=(11,4))
-
-plt.subplot(121)
-plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="Ensemble predictions")
-plt.title("learning_rate={}, n_estimators={}".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)
-
-plt.subplot(122)
-plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
-plt.title("learning_rate={}, n_estimators={}".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)
-
-save_fig("gbrt_learning_rate_plot")
-plt.show()
-
    
 
    
+The way we proceed in an iterative fashion is to
+
+
      
+
    1.  Initialize our estimate by \( f_0(x)=0 \).
      2. 
+
    2.  For \( m=1:M \), we
+
+
        
+ 
      1.  compute the negative gradient vector \( \boldsymbol{u}_m = -\partial C(\boldsymbol{y},\boldsymbol{f})/\partial \boldsymbol{f}(x) \) at $f(x) = f_{m-1}(x);
        2. 
+ 
      2.  fit the so-called base-learner to the negative gradient \( h_m(u_m,x) \);
        3. 
+ 
      3.  update the estimate \( f_m(x) = f_{m-1}(x)+\nu h_m(u_m,x) \);
        4. 
+
      
+
+
    3.  The final estimate is then \( f_M(x) = \sum_{m=1}^M\nu h_m(u_m,x) \).
      4. 
+
    
+
 
    
 
 
    
 
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs053.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs053.html
index b0c98925a..591f2cd60 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs053.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs053.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
               ('Final regressor code', 2, None, '___sec28'),
               ('Pros and cons of trees, pros', 2, None, '___sec29'),
               ('Disadvantages', 2, None, '___sec30'),
-              ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+              ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+               'Boosting, Meet the Jungle of Methods',
                2,
                None,
                '___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
                None,
                '___sec37'),
               ('Now Bagging', 2, None, '___sec38'),
-              ('Random forests', 2, None, '___sec39'),
-              ('Random Forest Algorithm', 2, None, '___sec40'),
+              ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+              ('Random forests', 2, None, '___sec40'),
+              ('Random Forest Algorithm', 2, None, '___sec41'),
               ('Random Forests Compared with other Methods on the Cancer Data',
                2,
                None,
-               '___sec41'),
+               '___sec42'),
               ('Compare  Bagging on Trees with Random Forests',
                2,
                None,
-               '___sec42'),
-              ('Feature Importance', 2, None, '___sec43'),
-              ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+               '___sec43'),
+              ('Bootstrap with Random Forests Instead of a Single Tree',
+               2,
+               None,
+               '___sec44'),
+              ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
               ('Adaptive boosting: AdaBoost, Basic Algorithm',
                2,
                None,
-               '___sec45'),
-              ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+               '___sec46'),
+              ('Basic Steps of AdaBoost', 2, None, '___sec47'),
               ('Figure to Illustrate the Iterative Classification Process',
                2,
                None,
-               '___sec47'),
-              ('AdaBoost Examples', 2, None, '___sec48'),
-              ('Gradient boosting: Basics', 2, None, '___sec49'),
-              ('Gradient Boosting, algorithm', 2, None, '___sec50'),
-              ('Gradient Boosting, Examples', 2, None, '___sec51'),
-              ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
-              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
-              ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+               '___sec48'),
+              ('AdaBoost Examples', 2, None, '___sec49'),
+              ('Gradient boosting: Basics', 2, None, '___sec50'),
+              ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+              ('Gradient Boosting, Examples', 2, None, '___sec52'),
+              ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+              ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
 end of tocinfo -->
 
 
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
       
  • Final regressor code
    • 
       
  • Pros and cons of trees, pros
    • 
       
  • Disadvantages
    • 
-      
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • 
+      
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
    • 
       
  • An Overview of Ensemble Methods
    • 
       
  • Bagging
    • 
       
  • More bagging
    • 
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
       
  • Using the Voting Classifier
    • 
       
  • Please, not the moons again! Voting and Bagging
    • 
       
  • Now Bagging
    • 
-      
  • Random forests
    • 
-      
  • Random Forest Algorithm
    • 
-      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
-      
  • Compare  Bagging on Trees with Random Forests
    • 
-      
  • Feature Importance
    • 
-      
  • Boosting, a Bird'e Eye
    • 
-      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
-      
  • Basic Steps of AdaBoost
    • 
-      
  • Figure to Illustrate the Iterative Classification Process
    • 
-      
  • AdaBoost Examples
    • 
-      
  • Gradient boosting: Basics
    • 
-      
  • Gradient Boosting, algorithm
    • 
-      
  • Gradient Boosting, Examples
    • 
-      
  • Gradient Boots with Early Stopping
    • 
-      
  • XGBoost: Extreme Gradient Boosting
    • 
-      
  • Xgboost on the Cancer Data
    • 
+      
  • Making our own Bagging with Bootstrap
    • 
+      
  • Random forests
    • 
+      
  • Random Forest Algorithm
    • 
+      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
+      
  • Compare  Bagging on Trees with Random Forests
    • 
+      
  • Bootstrap with Random Forests Instead of a Single Tree
    • 
+      
  • Boosting, a Bird'e Eye
    • 
+      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
+      
  • Basic Steps of AdaBoost
    • 
+      
  • Figure to Illustrate the Iterative Classification Process
    • 
+      
  • AdaBoost Examples
    • 
+      
  • Gradient boosting: Basics
    • 
+      
  • Gradient Boosting, algorithm
    • 
+      
  • Gradient Boosting, Examples
    • 
+      
  • Gradient Boots with Early Stopping
    • 
+      
  • XGBoost: Extreme Gradient Boosting
    • 
+      
  • Xgboost on the Cancer Data
    • 
 
         
       
@@ -234,66 +240,92 @@ MathJax.Hub.Config({
 
 
 
-
    Gradient Boots with Early Stopping 
    
+
    Gradient Boosting, Examples 
    
 
    
 
 
-
    from sklearn.model_selection import train_test_split
-from sklearn.metrics import mean_squared_error
+
    np.random.seed(42)
+X = np.random.rand(100, 1) - 0.5
+y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)
 
-X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=49)
+from sklearn.tree import DecisionTreeRegressor
 
-gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=120, random_state=42)
-gbrt.fit(X_train, y_train)
+tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg1.fit(X, y)
 
-errors = [mean_squared_error(y_val, y_pred)
-          for y_pred in gbrt.staged_predict(X_val)]
-bst_n_estimators = np.argmin(errors) + 1
+y2 = y - tree_reg1.predict(X)
+tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg2.fit(X, y2)
 
-gbrt_best = GradientBoostingRegressor(max_depth=2,n_estimators=bst_n_estimators, random_state=42)
-gbrt_best.fit(X_train, y_train)
+y3 = y2 - tree_reg2.predict(X)
+tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg3.fit(X, y3)
 
-min_error = np.min(errors)
-plt.figure(figsize=(11, 4))
+X_new = np.array([[0.8]])
+y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))
 
-plt.subplot(121)
-plt.plot(errors, "b.-")
-plt.plot([bst_n_estimators, bst_n_estimators], [0, min_error], "k--")
-plt.plot([0, 120], [min_error, min_error], "k--")
-plt.plot(bst_n_estimators, min_error, "ko")
-plt.text(bst_n_estimators, min_error*1.2, "Minimum", ha="center", fontsize=14)
-plt.axis([0, 120, 0, 0.01])
-plt.xlabel("Number of trees")
-plt.title("Validation error", fontsize=14)
+def plot_predictions(regressors, X, y, axes, label=None, style="r-", data_style="b.", data_label=None):
+    x1 = np.linspace(axes[0], axes[1], 500)
+    y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)
+    plt.plot(X[:, 0], y, data_style, label=data_label)
+    plt.plot(x1, y_pred, style, linewidth=2, label=label)
+    if label or data_label:
+        plt.legend(loc="upper center", fontsize=16)
+    plt.axis(axes)
 
-plt.subplot(122)
-plot_predictions([gbrt_best], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
-plt.title("Best model (%d trees)" % bst_n_estimators, fontsize=14)
+plt.figure(figsize=(11,11))
 
-save_fig("early_stopping_gbrt_plot")
+plt.subplot(321)
+plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h_1(x_1)$", style="g-", data_label="Training set")
+plt.ylabel("$y$", fontsize=16, rotation=0)
+plt.title("Residuals and tree predictions", fontsize=16)
+
+plt.subplot(322)
+plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1)$", data_label="Training set")
+plt.ylabel("$y$", fontsize=16, rotation=0)
+plt.title("Ensemble predictions", fontsize=16)
+
+plt.subplot(323)
+plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_2(x_1)$", style="g-", data_style="k+", data_label="Residuals")
+plt.ylabel("$y - h_1(x_1)$", fontsize=16)
+
+plt.subplot(324)
+plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1)$")
+plt.ylabel("$y$", fontsize=16, rotation=0)
+
+plt.subplot(325)
+plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_3(x_1)$", style="g-", data_style="k+")
+plt.ylabel("$y - h_1(x_1) - h_2(x_1)$", fontsize=16)
+plt.xlabel("$x_1$", fontsize=16)
+
+plt.subplot(326)
+plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$")
+plt.xlabel("$x_1$", fontsize=16)
+plt.ylabel("$y$", fontsize=16, rotation=0)
+
+save_fig("gradient_boosting_plot")
 plt.show()
 
+from sklearn.ensemble import GradientBoostingRegressor
 
-gbrt = GradientBoostingRegressor(max_depth=2, warm_start=True, random_state=42)
+gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0, random_state=42)
+gbrt.fit(X, y)
 
-min_val_error = float("inf")
-error_going_up = 0
-for n_estimators in range(1, 120):
-    gbrt.n_estimators = n_estimators
-    gbrt.fit(X_train, y_train)
-    y_pred = gbrt.predict(X_val)
-    val_error = mean_squared_error(y_val, y_pred)
-    if val_error < min_val_error:
-        min_val_error = val_error
-        error_going_up = 0
-    else:
-        error_going_up += 1
-        if error_going_up == 5:
-            break  # early stopping
+gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)
+gbrt_slow.fit(X, y)
 
+plt.figure(figsize=(11,4))
 
-print(gbrt.n_estimators)
-print("Minimum validation MSE:", min_val_error)
+plt.subplot(121)
+plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="Ensemble predictions")
+plt.title("learning_rate={}, n_estimators={}".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)
+
+plt.subplot(122)
+plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
+plt.title("learning_rate={}, n_estimators={}".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)
+
+save_fig("gbrt_learning_rate_plot")
+plt.show()
 
    
 
    
 
    
@@ -313,6 +345,7 @@ error_going_up = 54
   
  • 55
    • 
   
  • 56
    • 
+  
  • 57
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-bs.html b/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
index cd7951257..a15bd51fb 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
               ('Final regressor code', 2, None, '___sec28'),
               ('Pros and cons of trees, pros', 2, None, '___sec29'),
               ('Disadvantages', 2, None, '___sec30'),
-              ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+              ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+               'Boosting, Meet the Jungle of Methods',
                2,
                None,
                '___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
                None,
                '___sec37'),
               ('Now Bagging', 2, None, '___sec38'),
-              ('Random forests', 2, None, '___sec39'),
-              ('Random Forest Algorithm', 2, None, '___sec40'),
+              ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+              ('Random forests', 2, None, '___sec40'),
+              ('Random Forest Algorithm', 2, None, '___sec41'),
               ('Random Forests Compared with other Methods on the Cancer Data',
                2,
                None,
-               '___sec41'),
+               '___sec42'),
               ('Compare  Bagging on Trees with Random Forests',
                2,
                None,
-               '___sec42'),
-              ('Feature Importance', 2, None, '___sec43'),
-              ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+               '___sec43'),
+              ('Bootstrap with Random Forests Instead of a Single Tree',
+               2,
+               None,
+               '___sec44'),
+              ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
               ('Adaptive boosting: AdaBoost, Basic Algorithm',
                2,
                None,
-               '___sec45'),
-              ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+               '___sec46'),
+              ('Basic Steps of AdaBoost', 2, None, '___sec47'),
               ('Figure to Illustrate the Iterative Classification Process',
                2,
                None,
-               '___sec47'),
-              ('AdaBoost Examples', 2, None, '___sec48'),
-              ('Gradient boosting: Basics', 2, None, '___sec49'),
-              ('Gradient Boosting, algorithm', 2, None, '___sec50'),
-              ('Gradient Boosting, Examples', 2, None, '___sec51'),
-              ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
-              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
-              ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+               '___sec48'),
+              ('AdaBoost Examples', 2, None, '___sec49'),
+              ('Gradient boosting: Basics', 2, None, '___sec50'),
+              ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+              ('Gradient Boosting, Examples', 2, None, '___sec52'),
+              ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+              ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
 end of tocinfo -->
 
 
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
       
  • Final regressor code
    • 
       
  • Pros and cons of trees, pros
    • 
       
  • Disadvantages
    • 
-      
  • From a Single Tree to Many Trees, Meet the Jungle of Methods
    • 
+      
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
    • 
       
  • An Overview of Ensemble Methods
    • 
       
  • Bagging
    • 
       
  • More bagging
    • 
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
       
  • Using the Voting Classifier
    • 
       
  • Please, not the moons again! Voting and Bagging
    • 
       
  • Now Bagging
    • 
-      
  • Random forests
    • 
-      
  • Random Forest Algorithm
    • 
-      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
-      
  • Compare  Bagging on Trees with Random Forests
    • 
-      
  • Feature Importance
    • 
-      
  • Boosting, a Bird'e Eye
    • 
-      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
-      
  • Basic Steps of AdaBoost
    • 
-      
  • Figure to Illustrate the Iterative Classification Process
    • 
-      
  • AdaBoost Examples
    • 
-      
  • Gradient boosting: Basics
    • 
-      
  • Gradient Boosting, algorithm
    • 
-      
  • Gradient Boosting, Examples
    • 
-      
  • Gradient Boots with Early Stopping
    • 
-      
  • XGBoost: Extreme Gradient Boosting
    • 
-      
  • Xgboost on the Cancer Data
    • 
+      
  • Making our own Bagging with Bootstrap
    • 
+      
  • Random forests
    • 
+      
  • Random Forest Algorithm
    • 
+      
  • Random Forests Compared with other Methods on the Cancer Data
    • 
+      
  • Compare  Bagging on Trees with Random Forests
    • 
+      
  • Bootstrap with Random Forests Instead of a Single Tree
    • 
+      
  • Boosting, a Bird'e Eye
    • 
+      
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • 
+      
  • Basic Steps of AdaBoost
    • 
+      
  • Figure to Illustrate the Iterative Classification Process
    • 
+      
  • AdaBoost Examples
    • 
+      
  • Gradient boosting: Basics
    • 
+      
  • Gradient Boosting, algorithm
    • 
+      
  • Gradient Boosting, Examples
    • 
+      
  • Gradient Boots with Early Stopping
    • 
+      
  • XGBoost: Extreme Gradient Boosting
    • 
+      
  • Xgboost on the Cancer Data
    • 
 
         
       
@@ -253,7 +259,7 @@ MathJax.Hub.Config({
 
    [2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
    
 
    
 
    
-
    Nov 6, 2019
     
+
    Nov 7, 2019
     
 
    
 
    
 
@@ -277,7 +283,7 @@ MathJax.Hub.Config({
   
  • 9
    • 
   
  • 10
    • 
   
  • ...
    • 
-  
  • 56
    • 
+  
  • 57
    • 
   
  • »
    • 
 
 
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html b/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
index 00d97656f..496aefa10 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
@@ -148,7 +148,7 @@ MathJax.Hub.Config({
 
    [2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
    
 
    
 
     
    
-
    Nov 6, 2019
     
+
    Nov 7, 2019
     
 
    
 
    
 
@@ -698,6 +698,8 @@ os.system(cmd)
 
 
    
 
    Algorithms for Setting up Decision Trees 
    
+
+
    
 Two algorithms stand out in the set up of decision trees:
 
 
      
@@ -706,7 +708,7 @@ Two algorithms stand out in the set up of decision trees:
 
    
 
    
 
-We discuss both algorithms with applications here. The popular library -Scikit-Learn_ uses the CART algorithm. For classification problems you can use either the gini index or the entropy to split a tree in two branches.
+We discuss both algorithms with applications here. The popular library Scikit-Learn uses the CART algorithm. For classification problems you can use either the gini index or the entropy to split a tree in two branches.
 
    
 
 
@@ -1439,7 +1441,7 @@ However, by aggregating many decision trees, using methods like bagging, random
 
 
 
    
-
    From a Single Tree to Many Trees, Meet the Jungle of Methods 
    
+
    Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods 
    
 
 
    
 As stated above and seen in many of the examples discussed here about
@@ -1455,7 +1457,7 @@ machine learning algorithms or just use one of them to construct forests and jun
 
  •  Voting classifiers
    • 
 
  •  Bagging and Pasting
    • 
 
  •  Random forests
    • 
-
  •  Boosting methods
    • 
+
  •  Boosting methods, from adaptive to Extreme Gradient Boosting (XGBoost)
    • 
 
 
    
 
@@ -1714,7 +1716,68 @@ plt.show()
 
 
 
    
-
    Random forests 
    
+
    Making our own Bagging with Bootstrap 
    
+
    
+
+
+
    import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
+
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+    model = DecisionTreeRegressor(max_depth=5) 
+    y_pred = np.empty((y_test.shape[0], n_boostraps))
+    for i in range(n_boostraps):
+        x_, y_ = resample(X_train_scaled, y_train)
+        model.fit(x_, y_)
+        y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+    polydegree[degree] = degree
+    error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+    bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+    variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+    print('Polynomial degree:', degree)
+    print('Error:', error[degree])
+    print('Bias^2:', bias[degree])
+    print('Var:', variance[degree])
+    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
    
+
    
+
+
+
    
+
    Random forests 
    
 
 
    
 Random forests provide an improvement over bagged trees by way of a
@@ -1760,7 +1823,7 @@ this setting.
 
 
 
    
-
    Random Forest Algorithm 
    
+
    Random Forest Algorithm 
    
 The algorithm described here can be applied to both classification and regression problems.
 
 
    
@@ -1791,7 +1854,7 @@ We will grow of forest of say \( M \) trees.
 
 
 
    
-
    Random Forests Compared with other Methods on the Cancer Data 
    
+
    Random Forests Compared with other Methods on the Cancer Data 
    
 
    
 
 
@@ -1854,7 +1917,6 @@ accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=import scikitplot as skplt
 y_pred = Random_Forest_model.predict(X_test_scaled)
 skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
-#<matplotlib.axes._subplots.AxesSubplot object at 0x7fe967d64490>
 plt.show()
 y_probas = Random_Forest_model.predict_proba(X_test_scaled)
 skplt.metrics.plot_roc(y_test, y_probas)
@@ -1866,7 +1928,7 @@ plt.show()
 
 
 
    
-
    Compare  Bagging on Trees with Random Forests  
    
+
    Compare  Bagging on Trees with Random Forests  
    
 
    
 
 
@@ -1889,15 +1951,68 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
 
 
 
    
-
    Feature Importance 
    
+
    Bootstrap with Random Forests Instead of a Single Tree 
    
 
 
    
-Example will be added here.
+
+
+
    import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.ensemble import RandomForestRegressor
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+    model = RandomForestRegressor()
+    y_pred = np.empty((y_test.shape[0], n_boostraps))
+    for i in range(n_boostraps):
+        x_, y_ = resample(X_train_scaled, y_train)
+        model.fit(x_, y_)
+        y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+    polydegree[degree] = degree
+    error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+    bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+    variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+    print('Polynomial degree:', degree)
+    print('Error:', error[degree])
+    print('Bias^2:', bias[degree])
+    print('Var:', variance[degree])
+    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
    
 
    
 
 
 
    
-
    Boosting, a Bird'e Eye 
    
+
    Boosting, a Bird'e Eye 
    
 
 
    
 The basic idea is to combine weak classifiers in order to create a good
@@ -1914,7 +2029,7 @@ them with a factor.
 
 
 
    
-
    Adaptive boosting: AdaBoost, Basic Algorithm 
    
+
    Adaptive boosting: AdaBoost, Basic Algorithm 
    
 
 
    
 The algorithm here is rather straightforward. Assume that our weak
@@ -1938,7 +2053,7 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
 
 
 
    
-
    Basic Steps of AdaBoost 
    
+
    Basic Steps of AdaBoost 
    
 
 
    
 With the above definitions we are now ready to set up the algorithm for AdaBoost.
@@ -1973,18 +2088,18 @@ individually at each steps. The obersvations which were misclassified
 at iteration \( m-1 \) have a weight which is larger than those which were
 classified properly. As this proceeds, the observations which were
 difficult to classifiy correctly are given a larger influence. Each
-new classificatio step \( m \) is then forced to concentrate on those
+new classification step \( m \) is then forced to concentrate on those
 observations that are missed in the previous iterations.
 
    
 
 
 
    
-
    Figure to Illustrate the Iterative Classification Process 
    
+
    Figure to Illustrate the Iterative Classification Process 
    
 
    
 
 
 
    
-
    AdaBoost Examples 
    
+
    AdaBoost Examples 
    
 
 
    
 Using Scikit-Learn it is easy to appply the adaptive boosting algorithm, as done here.
@@ -2028,7 +2143,7 @@ plt.show()
 
 
 
    
-
    Gradient boosting: Basics 
    
+
    Gradient boosting: Basics 
    
 
 
    
 Gradient boosting is again a similar technique to Adapative boosting,
@@ -2043,7 +2158,7 @@ function was the least squares function.
 
 
 
    
-
    Gradient Boosting, algorithm 
    
+
    Gradient Boosting, algorithm 
    
 
 
    
 Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard least squares  function
@@ -2071,7 +2186,7 @@ The way we proceed in an iterative fashion is to
 
 
 
    
-
    Gradient Boosting, Examples 
    
+
    Gradient Boosting, Examples 
    
 
    
 
 
@@ -2162,7 +2277,7 @@ plt.show()
 
 
 
    
-
    Gradient Boots with Early Stopping 
    
+
    Gradient Boots with Early Stopping 
    
 
    
 
 
@@ -2227,7 +2342,7 @@ error_going_up = 0
 
 
 
    
-
    XGBoost: Extreme Gradient Boosting 
    
+
    XGBoost: Extreme Gradient Boosting 
    
 
 
    
 XGBoost or Extreme Gradient
@@ -2248,7 +2363,7 @@ It is now the algorithm which wins essentially all ML competitions!!!
 
 
 
    
-
    Xgboost on the Cancer Data 
    
+
    Xgboost on the Cancer Data 
    
 
    
 
 
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html b/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html
index 690ca8a77..b2c1e8ea1 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html
@@ -105,7 +105,8 @@ div { text-align: justify; text-justify: inter-word; }
               ('Final regressor code', 2, None, '___sec28'),
               ('Pros and cons of trees, pros', 2, None, '___sec29'),
               ('Disadvantages', 2, None, '___sec30'),
-              ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+              ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+               'Boosting, Meet the Jungle of Methods',
                2,
                None,
                '___sec31'),
@@ -119,34 +120,38 @@ div { text-align: justify; text-justify: inter-word; }
                None,
                '___sec37'),
               ('Now Bagging', 2, None, '___sec38'),
-              ('Random forests', 2, None, '___sec39'),
-              ('Random Forest Algorithm', 2, None, '___sec40'),
+              ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+              ('Random forests', 2, None, '___sec40'),
+              ('Random Forest Algorithm', 2, None, '___sec41'),
               ('Random Forests Compared with other Methods on the Cancer Data',
                2,
                None,
-               '___sec41'),
+               '___sec42'),
               ('Compare  Bagging on Trees with Random Forests',
                2,
                None,
-               '___sec42'),
-              ('Feature Importance', 2, None, '___sec43'),
-              ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+               '___sec43'),
+              ('Bootstrap with Random Forests Instead of a Single Tree',
+               2,
+               None,
+               '___sec44'),
+              ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
               ('Adaptive boosting: AdaBoost, Basic Algorithm',
                2,
                None,
-               '___sec45'),
-              ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+               '___sec46'),
+              ('Basic Steps of AdaBoost', 2, None, '___sec47'),
               ('Figure to Illustrate the Iterative Classification Process',
                2,
                None,
-               '___sec47'),
-              ('AdaBoost Examples', 2, None, '___sec48'),
-              ('Gradient boosting: Basics', 2, None, '___sec49'),
-              ('Gradient Boosting, algorithm', 2, None, '___sec50'),
-              ('Gradient Boosting, Examples', 2, None, '___sec51'),
-              ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
-              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
-              ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+               '___sec48'),
+              ('AdaBoost Examples', 2, None, '___sec49'),
+              ('Gradient boosting: Basics', 2, None, '___sec50'),
+              ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+              ('Gradient Boosting, Examples', 2, None, '___sec52'),
+              ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+              ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
 end of tocinfo -->
 
 
@@ -188,7 +193,7 @@ MathJax.Hub.Config({
 
    [2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
    
 
    
 
    
-
    Nov 6, 2019
     
+
    Nov 7, 2019
     
 
    
 
    
 









    
@@ -711,6 +716,8 @@ os.system(cmd)
 









    
 
 
    Algorithms for Setting up Decision Trees 
    
+
+
    
 Two algorithms stand out in the set up of decision trees:
 
 
      
@@ -718,7 +725,7 @@ Two algorithms stand out in the set up of decision trees:
 
    1.  The ID3 algorithm based on the computation of the information gain for classification
      2. 
 
    
 
-We discuss both algorithms with applications here. The popular library -Scikit-Learn_ uses the CART algorithm. For classification problems you can use either the gini index or the entropy to split a tree in two branches.
+We discuss both algorithms with applications here. The popular library Scikit-Learn uses the CART algorithm. For classification problems you can use either the gini index or the entropy to split a tree in two branches.
 
 
    
 









    
@@ -1439,7 +1446,7 @@ However, by aggregating many decision trees, using methods like bagging, random
 
    
 









    
 
-
    From a Single Tree to Many Trees, Meet the Jungle of Methods 
    
+
    Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods 
    
 
 
    
 As stated above and seen in many of the examples discussed here about
@@ -1455,7 +1462,7 @@ machine learning algorithms or just use one of them to construct forests and jun
 
  •  Voting classifiers
    • 
 
  •  Bagging and Pasting
    • 
 
  •  Random forests
    • 
-
  •  Boosting methods
    • 
+
  •  Boosting methods, from adaptive to Extreme Gradient Boosting (XGBoost)
    • 
 
 
 We discuss these methods here.
@@ -1709,7 +1716,67 @@ plt.show()
 
    
 









    
 
-
    Random forests 
    
+
    Making our own Bagging with Bootstrap 
    
+
    
+
+
+
    import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
+
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+    model = DecisionTreeRegressor(max_depth=5) 
+    y_pred = np.empty((y_test.shape[0], n_boostraps))
+    for i in range(n_boostraps):
+        x_, y_ = resample(X_train_scaled, y_train)
+        model.fit(x_, y_)
+        y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+    polydegree[degree] = degree
+    error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+    bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+    variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+    print('Polynomial degree:', degree)
+    print('Error:', error[degree])
+    print('Bias^2:', bias[degree])
+    print('Var:', variance[degree])
+    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
    
+
    
+









    
+
+
    Random forests 
    
 
 
    
 Random forests provide an improvement over bagged trees by way of a
@@ -1753,7 +1820,7 @@ this setting.
 
    
 









    
 
-
    Random Forest Algorithm 
    
+
    Random Forest Algorithm 
    
 The algorithm described here can be applied to both classification and regression problems.
 
 
    
@@ -1779,7 +1846,7 @@ We will grow of forest of say \( M \) trees.
 
 









    
 
-
    Random Forests Compared with other Methods on the Cancer Data 
    
+
    Random Forests Compared with other Methods on the Cancer Data 
    
 
    
 
 
@@ -1842,7 +1909,6 @@ accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=import scikitplot as skplt
 y_pred = Random_Forest_model.predict(X_test_scaled)
 skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
-#<matplotlib.axes._subplots.AxesSubplot object at 0x7fe967d64490>
 plt.show()
 y_probas = Random_Forest_model.predict_proba(X_test_scaled)
 skplt.metrics.plot_roc(y_test, y_probas)
@@ -1853,7 +1919,7 @@ plt.show()
 
    
 









    
 
-
    Compare  Bagging on Trees with Random Forests  
    
+
    Compare  Bagging on Trees with Random Forests  
    
 
    
 
 
@@ -1875,15 +1941,67 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
 
    
 









    
 
-
    Feature Importance 
    
+
    Bootstrap with Random Forests Instead of a Single Tree 
    
 
 
    
-Example will be added here.
 
+
+
    import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.ensemble import RandomForestRegressor
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+    model = RandomForestRegressor()
+    y_pred = np.empty((y_test.shape[0], n_boostraps))
+    for i in range(n_boostraps):
+        x_, y_ = resample(X_train_scaled, y_train)
+        model.fit(x_, y_)
+        y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+    polydegree[degree] = degree
+    error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+    bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+    variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+    print('Polynomial degree:', degree)
+    print('Error:', error[degree])
+    print('Bias^2:', bias[degree])
+    print('Var:', variance[degree])
+    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
    
 
    
 









    
 
-
    Boosting, a Bird'e Eye 
    
+
    Boosting, a Bird'e Eye 
    
 
 
    
 The basic idea is to combine weak classifiers in order to create a good
@@ -1900,7 +2018,7 @@ them with a factor.
 
    
 









    
 
-
    Adaptive boosting: AdaBoost, Basic Algorithm 
    
+
    Adaptive boosting: AdaBoost, Basic Algorithm 
    
 
 
    
 The algorithm here is rather straightforward. Assume that our weak
@@ -1922,7 +2040,7 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
 
    
 









    
 
-
    Basic Steps of AdaBoost 
    
+
    Basic Steps of AdaBoost 
    
 
 
    
 With the above definitions we are now ready to set up the algorithm for AdaBoost.
@@ -1956,18 +2074,18 @@ individually at each steps. The obersvations which were misclassified
 at iteration \( m-1 \) have a weight which is larger than those which were
 classified properly. As this proceeds, the observations which were
 difficult to classifiy correctly are given a larger influence. Each
-new classificatio step \( m \) is then forced to concentrate on those
+new classification step \( m \) is then forced to concentrate on those
 observations that are missed in the previous iterations.
 
 
    
 









    
 
-
    Figure to Illustrate the Iterative Classification Process 
    
+
    Figure to Illustrate the Iterative Classification Process 
    
 
 
    
 









    
 
-
    AdaBoost Examples 
    
+
    AdaBoost Examples 
    
 
 
    
 Using Scikit-Learn it is easy to appply the adaptive boosting algorithm, as done here.
@@ -2010,7 +2128,7 @@ plt.show()
 
    
 









    
 
-
    Gradient boosting: Basics 
    
+
    Gradient boosting: Basics 
    
 
 
    
 Gradient boosting is again a similar technique to Adapative boosting,
@@ -2025,7 +2143,7 @@ function was the least squares function.
 
    
 









    
 
-
    Gradient Boosting, algorithm 
    
+
    Gradient Boosting, algorithm 
    
 
 
    
 Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard least squares  function
@@ -2051,7 +2169,7 @@ The way we proceed in an iterative fashion is to
 
 









    
 
-
    Gradient Boosting, Examples 
    
+
    Gradient Boosting, Examples 
    
 
    
 
 
@@ -2141,7 +2259,7 @@ plt.show()
 
    
 









    
 
-
    Gradient Boots with Early Stopping 
    
+
    Gradient Boots with Early Stopping 
    
 
    
 
 
@@ -2205,7 +2323,7 @@ error_going_up = 0
 
    
 









    
 
-
    XGBoost: Extreme Gradient Boosting 
    
+
    XGBoost: Extreme Gradient Boosting 
    
 
 
    
 XGBoost or Extreme Gradient
@@ -2226,7 +2344,7 @@ It is now the algorithm which wins essentially all ML competitions!!!
 
    
 









    
 
-
    Xgboost on the Cancer Data 
    
+
    Xgboost on the Cancer Data 
    
 
    
 
 
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees.html b/doc/pub/DecisionTrees/html/DecisionTrees.html
index 186eea8a1..ede4f121d 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees.html
@@ -110,7 +110,8 @@ div { text-align: justify; text-justify: inter-word; }
               ('Final regressor code', 2, None, '___sec28'),
               ('Pros and cons of trees, pros', 2, None, '___sec29'),
               ('Disadvantages', 2, None, '___sec30'),
-              ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+              ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+               'Boosting, Meet the Jungle of Methods',
                2,
                None,
                '___sec31'),
@@ -124,34 +125,38 @@ div { text-align: justify; text-justify: inter-word; }
                None,
                '___sec37'),
               ('Now Bagging', 2, None, '___sec38'),
-              ('Random forests', 2, None, '___sec39'),
-              ('Random Forest Algorithm', 2, None, '___sec40'),
+              ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+              ('Random forests', 2, None, '___sec40'),
+              ('Random Forest Algorithm', 2, None, '___sec41'),
               ('Random Forests Compared with other Methods on the Cancer Data',
                2,
                None,
-               '___sec41'),
+               '___sec42'),
               ('Compare  Bagging on Trees with Random Forests',
                2,
                None,
-               '___sec42'),
-              ('Feature Importance', 2, None, '___sec43'),
-              ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+               '___sec43'),
+              ('Bootstrap with Random Forests Instead of a Single Tree',
+               2,
+               None,
+               '___sec44'),
+              ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
               ('Adaptive boosting: AdaBoost, Basic Algorithm',
                2,
                None,
-               '___sec45'),
-              ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+               '___sec46'),
+              ('Basic Steps of AdaBoost', 2, None, '___sec47'),
               ('Figure to Illustrate the Iterative Classification Process',
                2,
                None,
-               '___sec47'),
-              ('AdaBoost Examples', 2, None, '___sec48'),
-              ('Gradient boosting: Basics', 2, None, '___sec49'),
-              ('Gradient Boosting, algorithm', 2, None, '___sec50'),
-              ('Gradient Boosting, Examples', 2, None, '___sec51'),
-              ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
-              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
-              ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+               '___sec48'),
+              ('AdaBoost Examples', 2, None, '___sec49'),
+              ('Gradient boosting: Basics', 2, None, '___sec50'),
+              ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+              ('Gradient Boosting, Examples', 2, None, '___sec52'),
+              ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+              ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+              ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
 end of tocinfo -->
 
 
@@ -193,7 +198,7 @@ MathJax.Hub.Config({
 
    [2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
    
 
    
 
    
-
    Nov 6, 2019
     
+
    Nov 7, 2019
     
 
    
 
    
 









    
@@ -716,6 +721,8 @@ os.system(cmd)
 









    
 
 
    Algorithms for Setting up Decision Trees 
    
+
+
    
 Two algorithms stand out in the set up of decision trees:
 
 
      
@@ -723,7 +730,7 @@ Two algorithms stand out in the set up of decision trees:
 
    1.  The ID3 algorithm based on the computation of the information gain for classification
      2. 
 
    
 
-We discuss both algorithms with applications here. The popular library -Scikit-Learn_ uses the CART algorithm. For classification problems you can use either the gini index or the entropy to split a tree in two branches.
+We discuss both algorithms with applications here. The popular library Scikit-Learn uses the CART algorithm. For classification problems you can use either the gini index or the entropy to split a tree in two branches.
 
 
    
 









    
@@ -1444,7 +1451,7 @@ However, by aggregating many decision trees, using methods like bagging, random
 
    
 









    
 
-
    From a Single Tree to Many Trees, Meet the Jungle of Methods 
    
+
    Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods 
    
 
 
    
 As stated above and seen in many of the examples discussed here about
@@ -1460,7 +1467,7 @@ machine learning algorithms or just use one of them to construct forests and jun
 
  •  Voting classifiers
    • 
 
  •  Bagging and Pasting
    • 
 
  •  Random forests
    • 
-
  •  Boosting methods
    • 
+
  •  Boosting methods, from adaptive to Extreme Gradient Boosting (XGBoost)
    • 
 
 
 We discuss these methods here.
@@ -1714,7 +1721,67 @@ plt.show()
 
    
 









    
 
-
    Random forests 
    
+
    Making our own Bagging with Bootstrap 
    
+
    
+
+
+
    import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
+
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+    model = DecisionTreeRegressor(max_depth=5) 
+    y_pred = np.empty((y_test.shape[0], n_boostraps))
+    for i in range(n_boostraps):
+        x_, y_ = resample(X_train_scaled, y_train)
+        model.fit(x_, y_)
+        y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+    polydegree[degree] = degree
+    error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+    bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+    variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+    print('Polynomial degree:', degree)
+    print('Error:', error[degree])
+    print('Bias^2:', bias[degree])
+    print('Var:', variance[degree])
+    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
    
+
    
+









    
+
+
    Random forests 
    
 
 
    
 Random forests provide an improvement over bagged trees by way of a
@@ -1758,7 +1825,7 @@ this setting.
 
    
 









    
 
-
    Random Forest Algorithm 
    
+
    Random Forest Algorithm 
    
 The algorithm described here can be applied to both classification and regression problems.
 
 
    
@@ -1784,7 +1851,7 @@ We will grow of forest of say \( M \) trees.
 
 









    
 
-
    Random Forests Compared with other Methods on the Cancer Data 
    
+
    Random Forests Compared with other Methods on the Cancer Data 
    
 
    
 
 
@@ -1847,7 +1914,6 @@ accuracy = cross_validate(Random_Forest_mode
 import scikitplot as skplt
 y_pred = Random_Forest_model.predict(X_test_scaled)
 skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
-#<matplotlib.axes._subplots.AxesSubplot object at 0x7fe967d64490>
 plt.show()
 y_probas = Random_Forest_model.predict_proba(X_test_scaled)
 skplt.metrics.plot_roc(y_test, y_probas)
@@ -1858,7 +1924,7 @@ plt.show()
 
    
 









    
 
-
    Compare  Bagging on Trees with Random Forests  
    
+
    Compare  Bagging on Trees with Random Forests  
    
 
    
 
 
@@ -1880,15 +1946,67 @@ np.sum(y_pred =
 
    
 









    
 
-
    Feature Importance 
    
+
    Bootstrap with Random Forests Instead of a Single Tree 
    
 
 
    
-Example will be added here.
 
+
+
    import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.ensemble import RandomForestRegressor
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+    model = RandomForestRegressor()
+    y_pred = np.empty((y_test.shape[0], n_boostraps))
+    for i in range(n_boostraps):
+        x_, y_ = resample(X_train_scaled, y_train)
+        model.fit(x_, y_)
+        y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+    polydegree[degree] = degree
+    error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+    bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+    variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+    print('Polynomial degree:', degree)
+    print('Error:', error[degree])
+    print('Bias^2:', bias[degree])
+    print('Var:', variance[degree])
+    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
    
 
    
 









    
 
-
    Boosting, a Bird'e Eye 
    
+
    Boosting, a Bird'e Eye 
    
 
 
    
 The basic idea is to combine weak classifiers in order to create a good
@@ -1905,7 +2023,7 @@ them with a factor.
 
    
 









    
 
-
    Adaptive boosting: AdaBoost, Basic Algorithm 
    
+
    Adaptive boosting: AdaBoost, Basic Algorithm 
    
 
 
    
 The algorithm here is rather straightforward. Assume that our weak
@@ -1927,7 +2045,7 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
 
    
 









    
 
-
    Basic Steps of AdaBoost 
    
+
    Basic Steps of AdaBoost 
    
 
 
    
 With the above definitions we are now ready to set up the algorithm for AdaBoost.
@@ -1961,18 +2079,18 @@ individually at each steps. The obersvations which were misclassified
 at iteration \( m-1 \) have a weight which is larger than those which were
 classified properly. As this proceeds, the observations which were
 difficult to classifiy correctly are given a larger influence. Each
-new classificatio step \( m \) is then forced to concentrate on those
+new classification step \( m \) is then forced to concentrate on those
 observations that are missed in the previous iterations.
 
 
    
 









    
 
-
    Figure to Illustrate the Iterative Classification Process 
    
+
    Figure to Illustrate the Iterative Classification Process 
    
 
 
    
 









    
 
-
    AdaBoost Examples 
    
+
    AdaBoost Examples 
    
 
 
    
 Using Scikit-Learn it is easy to appply the adaptive boosting algorithm, as done here.
@@ -2015,7 +2133,7 @@ plt.show()
 
    
 









    
 
-
    Gradient boosting: Basics 
    
+
    Gradient boosting: Basics 
    
 
 
    
 Gradient boosting is again a similar technique to Adapative boosting,
@@ -2030,7 +2148,7 @@ function was the least squares function.
 
    
 









    
 
-
    Gradient Boosting, algorithm 
    
+
    Gradient Boosting, algorithm 
    
 
 
    
 Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard least squares  function
@@ -2056,7 +2174,7 @@ The way we proceed in an iterative fashion is to
 
 









    
 
-
    Gradient Boosting, Examples 
    
+
    Gradient Boosting, Examples 
    
 
    
 
 
@@ -2146,7 +2264,7 @@ plt.show()
 
    
 









    
 
-
    Gradient Boots with Early Stopping 
    
+
    Gradient Boots with Early Stopping 
    
 
    
 
 
@@ -2210,7 +2328,7 @@ error_going_up = XGBoost: Extreme Gradient Boosting 
+
    XGBoost: Extreme Gradient Boosting 
    
 
 
    
 XGBoost or Extreme Gradient
@@ -2231,7 +2349,7 @@ It is now the algorithm which wins essentially all ML competitions!!!
 
    
 









    
 
-
    Xgboost on the Cancer Data 
    
+
    Xgboost on the Cancer Data 
    
 
    
 
 
diff --git a/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb b/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
index b0eaa13f3..59ba61dcb 100644
--- a/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
+++ b/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
@@ -10,7 +10,7 @@
     "  \n",
     "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n",
     "\n",
-    "Date: **Nov 6, 2019**\n",
+    "Date: **Nov 7, 2019**\n",
     "\n",
     "Copyright 1999-2019, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license\n",
     "\n",
@@ -603,12 +603,13 @@
    "metadata": {},
    "source": [
     "## Algorithms for Setting up Decision Trees\n",
+    "\n",
     "Two algorithms stand out in the set up of decision trees:\n",
     "1. The CART (Classification And Regression Tree) algorithm for both classification and regression\n",
     "\n",
     "2. The ID3 algorithm based on the computation of the information gain for classification\n",
     "\n",
-    "We discuss both algorithms with applications here. The popular library -Scikit-Learn_ uses the CART algorithm. For classification problems you can use either the **gini** index or the **entropy** to split a tree in two branches.\n",
+    "We discuss both algorithms with applications here. The popular library **Scikit-Learn** uses the CART algorithm. For classification problems you can use either the **gini** index or the **entropy** to split a tree in two branches.\n",
     "\n",
     "## The CART algorithm for Classification\n",
     "\n",
@@ -1397,7 +1398,7 @@
     "However, by aggregating many decision trees, using methods like bagging, random forests, and boosting, the predictive performance of trees can be substantially improved. \n",
     "\n",
     "\n",
-    "## From a Single Tree to Many Trees, Meet the Jungle of Methods\n",
+    "## Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods\n",
     "\n",
     "As stated above and seen in many of the examples discussed here about\n",
     "a single decision tree, we often end up overfitting our training\n",
@@ -1412,7 +1413,7 @@
     "\n",
     "3. Random forests\n",
     "\n",
-    "4. Boosting methods\n",
+    "4. Boosting methods, from adaptive to Extreme Gradient Boosting (XGBoost)\n",
     "\n",
     "We discuss these methods here.\n",
     "\n",
@@ -1726,6 +1727,75 @@
     "plt.show()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Making our own Bagging with Bootstrap"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "import numpy as np\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.pipeline import make_pipeline\n",
+    "from sklearn.utils import resample\n",
+    "from sklearn.tree import DecisionTreeRegressor\n",
+    "\n",
+    "\n",
+    "np.random.seed(2018)\n",
+    "\n",
+    "n = 40\n",
+    "n_boostraps = 100\n",
+    "maxdegree = 14\n",
+    "\n",
+    "# Make data set.\n",
+    "x = np.linspace(-3, 3, n).reshape(-1, 1)\n",
+    "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n",
+    "error = np.zeros(maxdegree)\n",
+    "bias = np.zeros(maxdegree)\n",
+    "variance = np.zeros(maxdegree)\n",
+    "polydegree = np.zeros(maxdegree)\n",
+    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
+    "\n",
+    "from sklearn.preprocessing import StandardScaler\n",
+    "scaler = StandardScaler()\n",
+    "scaler.fit(X_train)\n",
+    "X_train_scaled = scaler.transform(X_train)\n",
+    "X_test_scaled = scaler.transform(X_test)\n",
+    "\n",
+    "for degree in range(maxdegree):\n",
+    "    model = DecisionTreeRegressor(max_depth=5) \n",
+    "    y_pred = np.empty((y_test.shape[0], n_boostraps))\n",
+    "    for i in range(n_boostraps):\n",
+    "        x_, y_ = resample(X_train_scaled, y_train)\n",
+    "        model.fit(x_, y_)\n",
+    "        y_pred[:, i] = model.predict(X_test_scaled).ravel()\n",
+    "\n",
+    "    polydegree[degree] = degree\n",
+    "    error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )\n",
+    "    bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )\n",
+    "    variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )\n",
+    "    print('Polynomial degree:', degree)\n",
+    "    print('Error:', error[degree])\n",
+    "    print('Bias^2:', bias[degree])\n",
+    "    print('Var:', variance[degree])\n",
+    "    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))\n",
+    "\n",
+    "plt.plot(polydegree, error, label='Error')\n",
+    "plt.plot(polydegree, bias, label='bias')\n",
+    "plt.plot(polydegree, variance, label='Variance')\n",
+    "plt.legend()\n",
+    "plt.show()"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -1802,7 +1872,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 25,
    "metadata": {
     "collapsed": false
    },
@@ -1867,7 +1937,6 @@
     "import scikitplot as skplt\n",
     "y_pred = Random_Forest_model.predict(X_test_scaled)\n",
     "skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)\n",
-    "#\n",
     "plt.show()\n",
     "y_probas = Random_Forest_model.predict_proba(X_test_scaled)\n",
     "skplt.metrics.plot_roc(y_test, y_probas)\n",
@@ -1885,7 +1954,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 26,
    "metadata": {
     "collapsed": false
    },
@@ -1898,7 +1967,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 27,
    "metadata": {
     "collapsed": false
    },
@@ -1917,11 +1986,75 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Feature Importance\n",
+    "## Bootstrap with Random Forests Instead of a Single Tree"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
     "\n",
-    "Example will be added here.\n",
+    "import matplotlib.pyplot as plt\n",
+    "import numpy as np\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.pipeline import make_pipeline\n",
+    "from sklearn.utils import resample\n",
+    "from sklearn.ensemble import RandomForestRegressor\n",
     "\n",
+    "np.random.seed(2018)\n",
     "\n",
+    "n = 40\n",
+    "n_boostraps = 100\n",
+    "maxdegree = 14\n",
+    "\n",
+    "# Make data set.\n",
+    "x = np.linspace(-3, 3, n).reshape(-1, 1)\n",
+    "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n",
+    "error = np.zeros(maxdegree)\n",
+    "bias = np.zeros(maxdegree)\n",
+    "variance = np.zeros(maxdegree)\n",
+    "polydegree = np.zeros(maxdegree)\n",
+    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
+    "\n",
+    "from sklearn.preprocessing import StandardScaler\n",
+    "scaler = StandardScaler()\n",
+    "scaler.fit(X_train)\n",
+    "X_train_scaled = scaler.transform(X_train)\n",
+    "X_test_scaled = scaler.transform(X_test)\n",
+    "\n",
+    "for degree in range(maxdegree):\n",
+    "    model = RandomForestRegressor()\n",
+    "    y_pred = np.empty((y_test.shape[0], n_boostraps))\n",
+    "    for i in range(n_boostraps):\n",
+    "        x_, y_ = resample(X_train_scaled, y_train)\n",
+    "        model.fit(x_, y_)\n",
+    "        y_pred[:, i] = model.predict(X_test_scaled).ravel()\n",
+    "\n",
+    "    polydegree[degree] = degree\n",
+    "    error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )\n",
+    "    bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )\n",
+    "    variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )\n",
+    "    print('Polynomial degree:', degree)\n",
+    "    print('Error:', error[degree])\n",
+    "    print('Bias^2:', bias[degree])\n",
+    "    print('Var:', variance[degree])\n",
+    "    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))\n",
+    "\n",
+    "plt.plot(polydegree, error, label='Error')\n",
+    "plt.plot(polydegree, bias, label='bias')\n",
+    "plt.plot(polydegree, variance, label='Variance')\n",
+    "plt.legend()\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
     "## Boosting, a Bird'e Eye\n",
     "\n",
     "The basic idea is to combine weak classifiers in order to create a good\n",
@@ -2003,7 +2136,7 @@
     "at iteration $m-1$ have a weight which is larger than those which were\n",
     "classified properly. As this proceeds, the observations which were\n",
     "difficult to classifiy correctly are given a larger influence. Each\n",
-    "new classificatio step $m$ is then forced to concentrate on those\n",
+    "new classification step $m$ is then forced to concentrate on those\n",
     "observations that are missed in the previous iterations.\n",
     "\n",
     "## Figure to Illustrate the Iterative Classification Process\n",
@@ -2016,7 +2149,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 29,
    "metadata": {
     "collapsed": false
    },
@@ -2106,7 +2239,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 30,
    "metadata": {
     "collapsed": false
    },
@@ -2205,7 +2338,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 31,
    "metadata": {
     "collapsed": false
    },
@@ -2295,7 +2428,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 32,
    "metadata": {
     "collapsed": false
    },
diff --git a/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz b/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz
index 8efd3d8701d5a25a5531b18aeb603eda9459f8f6..10e484876fcdebedb2961f27b53b0c62b0327ab4 100644
GIT binary patch
delta 29
lcmZ4cQ*iB1K{okr4hGImjcl!KjIC@;t!&I&*;tm>005WK2=V{`

delta 29
kcmZ4cQ*iB1K{okr4u(MIMz&Tq##T0_RyO9XY%I%b0F{CW%K!iX

diff --git a/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf b/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf
index 4556a78e2df3f37e3a471894d9f3516e2befe9f0..c5c80b2157ed4b94ef2cc0f7e4e21f3771c97451 100644
GIT binary patch
delta 38640
zcmZsiQ+TFLuxOKfu_v}|+n!)zI}>YSCvR-qm>3h==ER=ZwtfEnoPF-jRaf`ZSAA2x
zYE{h@kfbGX1Bv7{M!LeKzzwB(%kTEBHq*K8+L)YlFML1tuAHLMmjyHS-7@|rp2
z)Bf9KlBA>9r~P{KiGj9MNp)vxgGH(h@Y_CvNv2f%p89?u{+M&4_(bgY{C!M9r@+)c
z@ohvRQ*1?rS~3@*aD4wW{+<7$&h5dz#xB>7;Plh%(mZ!@ARuDewP7A$6Lb@wK@Pr+=5xA<(Vl;?|~KK!b)Yj4Ac$=$f3%lTo4KiHZe-D8&h
zZ&iPy=4}&!?4oHrh+-mfz0U3aA?Br4pE9KkUOoQA`)4o%!?3Jn#@*rc-oB1X<-&;}
zFtp$L)kbK}up>dBL=gg%RWWs46`?`9St}1-u+^&5b0Nu!flptL;2+TxM3J%4n2$zL
zzyz5oP^xiC4g8?3V#C+=OiivWzg~uma>_VwqG_O8H}kHhb`J!vkR?Q9=OZNce1A*8
za_pJ(8KM8eYq!v`DQ$E9+;bb*I`r!-HzEyTh%z;VUmsga{E7nnI(xxb=stKltO)$6
z)axVZ{N_*6{()<|PEUMwX1ja;Qf5aEliTxG`MfB}lp{L8s0fu!oB;#pCxh+YU*+fxRzVdo;dw>=MODP3QdGkYD|1qO#^O~xQ7i|9+4=6&sd=cfOb@78@M8?D|SoV{8#nj6XHtLO&+N!3kHu}C=UpO>dEE!x+g
zDP1KP8e~g$OlP}L
zYw-RMj_}IxG&-}pT1XoOy?NZ!{n;?H9)R154OYz}D!ja%G{e*}nmvoPByr(g$~KwC
z4=SGL>F}=wT-&3k)WC8e4wu4QL6?V(JfDoVSnA2@zs0C@uZEEp<^#Y*=p-fe1O=O7
zPmJg)BUzG9jFr8OQeW2HFCSo?#Gq}lmVb;UR75%i!Kk&bPRwA9;&&mFXv3gRcXalK
zMY*#berjYGBQsR$_S$wk)Q8(Pt!1~?pSqrrjdw=^bb{-@ersvcY*hg~DM|u{ukmXv
zU5H_Br-rGXwa$lnDvH;J_{SN%Q=%w@8uH@_d}JI%uNn-Woyf4s29jNaHX_H2dJ`NB
z)r7@x)ib+`+NL0^srlnOP8HXcVJlpHk??7nAi|dc562_9=SLZ8l26l9rhF{zWT5&X
zZZjEZQ&Jy#MG4;RE{*(M=~CSRQWfH1g;ww1WknS-5@;F(PlGfd2*Y|Y{e^n|xb3x{uqUjihnevHgJD-b;mq1bBH9oGT^lZH^XIf{pj^MKG*6u`wSj_*YDhnY
z*>85nM!4lp_~Ar!oZ$f;NHt?8TpIMY>Z22&B^(0Lbv~Lp_|ymS4zc^s05vWxw*zAB(P
zWPlWMmY_Qn;O4cmbxr*`F*htIXO=LePC=^+R~Qr}3|-o-p(bl1Bv$=?IZDWUR1F6}
z&sAS$=(%5qBx${H4If`WfkV~gN3_nFne0iqo1O8`M7W)(MAXwYTWr{&-Fw$g!5gzR
zY`~Gz90$N}LHkUf*eFCZN~E6IDC|n=l|rg(8QV5pqA!`z=AXFVr(p&{#Z^p-eq%U%
z^O6q1&}k=_S02JZr6@-5ZBKa+{qQ>=!B=0FMqwdxJ%?K-qLbWl|I166aQCmsw`BjGu^O6fFpFP`qYS0)<5>@})
zSS+J^a@q*y&S92n6jM)@0^opL|A1}0R6&@pEofz-`Uo&te2olq1wp06arsw__Ot*Y
z9FiLht?Px>?5R)lt4(!*hW_D^(4L>&Zwws}&{(?@^DV6D?7?i`8Idu(e}{&$_3SM!
z-y&=l+w7k#Gfd-oAq2#C?(-wqb}0Gg>JlfWLuf9LbzL)vM6LJ6h-u!rI3r_^};xRnJed+m4Cn?BBjxo-@Y;SnQF98-DA2QYWaZU!43o%kwyV
z0B~orzRX}NG>X4018Dqc1B3jr{sy;c8US0J>50u`xSeKUE0oeXS
z;SBE$MImC;byhjA)uc!n_)qUIp95cD2)(z4L4RSw{=$H_{Dpx~_D<`=g`@?FaVDJB
zxUMg2w!dZ8q3()8oXaDZt8!m(Tr4bcgf+tG{HBbhRXG@leS6p#3gn6f(;^O7W?|}bGm
zXGXSk$p)bjCDFh)!F1xGUZemdec{q|{aEXy!v_BNL4TLSbut^`X48Fmj>Gh!tQ@FLRJk%l{oIO_Q9LHu6G@{^3hL}&%#wC^xi{3}
zRXLRk5778v3O`NoTO?tfQ>`s&&m#)^3zNAJ>tRZ;@&v=uVEOmxGR)ds#9rG_Tb%UM
z#=6EZau=OQd~@OE#TEy+XxQp};gH$JhZwk6U!;j%q_0`jM%W7D!&Krv3L)Q4y2M0&
zR6)G!|1-sZ4dFodb%4^g&3`IJfXJREK!c%viy+gda#YT
zp*Ud{g%XFBRwA#%ayddDULE7MP2xtFX|TcLixUTpSjmo@(-H%IyabWirmvO!M#+n~
zh73(Vyn1lmNMggPWo-H}Ui5R8rq#?;Vk@e+)oqTs`7$beJEn+lQ|C;}gLm;VM-$sI
zp`Owh{!C=-4v`&ye3KSX<_i~dD2a)1UN%KCqZ)z&(?t5F4hki(T@%Yd&;SJ^i?Wjs
zR3}8<>eO?-bEyLCfrO{0)uWFFqYY=cgoI00-_!V8*G(ZUcVBAv+FW5$
zWIGn^#4|ZeyqIOBy+rF4ZJXzhO(;ALx#zzYOLv(Q4(WwD0*v(7@k4g{gx7BZl-gfAktjU1^us?h6eV;sQ#u2aSfi&CM*aBmFPe;Km>OLX18xx+5fB#e)9g;
z?CQ7NR9!h3D0t!I1W82s!_Af~D>_!O15wPtTcrrN6E;Fbm4*Wk|EnNSWoT?WlB(YuG&zy+z#mXAJ4UeN(ZRn?v%NkE??Nu=O2t$qR1dsXIE?Hx0WJR&
zWQI}W8DsS@TY-cW0p!JuCjl4L;KL%jYT68dj9KNxcS|d{kQ(vk!ZB>uUfV9K;l8T7
zlcu>>2}5HzKA#SUpy}m5FlXc5?K{Fik2l@6(FPA~30jO-xj|y9Q8tGk{3VV7|0tcB
z%Ui00c<#&cYM~RI@KF}>H
z>6vKhD14)=UV`c?VgTcJiK~IfQ%~aOz@H1lzF&tg2|iTr5#p--Y&{-`6f-fZ^mO`g
z8aUxoD_xp5b6xBKd&`B%zw&hGI(Y=V)2+X@8LVMB1#nBrQMhT+5JgV!z?F-huV+qr
zgsYf7yLtSSMf65=m~RYlYxP?0
zbV_RUgTy`+qYzD*JH-f!XFmgwGAktNbv1dDg{5>K3;JWe!w(3xN+k2eUKbbBWEA-`
zvYxl}I1-II03)4?1z2Gk@DZ>=NhcUPn3W6_mA}%#Y)HJ@25?@
zOnZ9C&|Z^4CGykG<^F7eyLmB7+ZqolvWEn@)~QEwjO93Ud++iOb!d#Q$=o9eK(UEO
z`x;z2^S|(4CvHi?zNJ^LLAXYstlpnSG$A((;Dn!0J-e~4Ehmu>smLH6HunDu*D;!M
zaqC=&-A9_w$KZUiX4-!DDlSpB|G=OkUEaVFi8z|ghyM_(PHav-E~jD8PJ6K7@0-YB
zNGkjI6zy)K4)(2S-YkAy?kt{p(+0B9HBcZBJqE}`(O`d%FBgz4f+srN7Jp{NdO55K
zbD!v5ej@8x!H_lszvn>>P?AhO08Tr5fX-5*C?0=bQJ$(j?JB)
zz}lE|a5RXI6n>9`Vc%(meUegau<%?j6Ii&g)#?T7A*-tJq&P%I$)SHI%3yo$dYiSq
zk>XZYR8Phzdg*C!Y*%EYdFou1O$eR;R=p!N;KETjRENkP7jl=0;jW*fb%i5#;WHG`Y5ekP;}
z57P;8g^@EZC|G!{A$Hjti=swd+UQ8C+rQFdDQDLAWV2deH}YG2XX~Nh>_bxcMg4=s
z&3Ir_ON~)W{XXSxZ=PF|X>g@bIc&V}D|~P6g7{~~kH3oes%ce*7r^LwB_XbbCYN$Z
zcrwix)l-&OsKwPNkB@Z;c?{nL!JB-;()Gh1!!9jT@43M+ew;bU&S!>QC`m|`>$=@t
ztvpaAxuFPJN=m{w4-@ZVk^sunfDEYCUb@hHw+H?4wc7*S5M5g=(!yre-v&RW6=4|l
z56{wL24olH!uJHJdO&755cQ7|g12>-7!#O7ujM2nc59@-Yv$Ti$0Aq#Y|LMEC7Sy5
z55apTA9Vr^P(YNu7g;v!#GJFJ$SLl=(XJ>msYT^3hYwFXVnFfmzl;{mOiDJK8Hr_1
zo-7i)bT_Mn#1o
zYAMEX$JGVHCzObh3wbS7u;7>^<$Cn8D(Wt6v)jl<#mv1oO-jfy6aS$V+kLJoJ?`4V
zgn_aDA^r557)A7a1t>tndrm6l}olj}433Gr+!zBh&QZ)}Y`FE}~nq16$&P
znCaIONco-{rV!y8YO%2BPcdWQ^mE@sW%pZw*>@vzY$GKcQ?(L}1
zF9hl?DkbW)nmZjCoPl*Un`tdCZ2JY0X1GO0Ie#@buJkiSdw5SG5{Ya5JNJ?F`2f<5
zAw)L(d0@V52avY+DalYp;&K=d@>guKf#qaJ`>~HStw?rz-GQV
zB{aQ=J`FxJ+RBqiowb>es@3_!X0dNZ>$4fU_wd0N76tJ9aglSeydTM3cV_oAi=Y`-
ze?_$0@->I?a~mW07Io2^${_2d*jm)T`WMjr6V*?vX{7dp!$B6*W#8+G_Pg>(2nCzp
z7VwctNMym{S>1&qPr(PnArau}BJD43bV?|p&n8TmD0FbL3qqV34Wq_C7hz1cM*(3p
z{v6@?p2Cj&m3tvkxcVP?-KSmC{k+Usx6Bv5M|8tGvL=&)_7RG3!j>ctUXfgqYZ9l!
zOzD4;)T>eg6H*Pc=Skm0Y|u~c=mggp1Vl2VsCN4>pD_NFl!J}nO|)n)^B2u@AVL?G
zhl+2#Tb!;Hq(J1;ah=(V5+5&!_8}M^kha`*T_OtODV!D0^~#7
zkJjRye&!V12OG+zyZlq-DIb<=!
z2Xn5(aU8%Cm_U#nxdH(r+^7Cg?Mu|lDX6j#L&gw|eR3f~V{6LGlekbAi+a&x2P9rI
zpAihdrgd6d&RwqqR+BJ@P*f7<=z&&gJlqhuwOhA)M@P&nJK?Qq#P(k3PZ$Mq{
zM0YbjWxNOC5T7KO7zvMCJiVG{b{N%`eGGxbi;*hais9jbcD$ZfVEB4^;@`%9rkj^7BRU%>3wiS8M4R>;KKcicqk&G#9G=<{E%
z6aBIIs_&57!-<=lD72;b=>f1on>?3!8_1)AU!1^!JUh_OjS{f$6s%WdN&L@;+Iedt+NK
zPaC3h3*y0mnVHi=Kwm1rINVw;nhgUxv)LBKYYq`-AAha&w?7aHW!H`nSml|LKiPir
zH}kexpX;6?|07`l;sqF}vJs~|gbRv#uW}VJs*EHw9rEq#XV8YUkuXuPJ9h(voeE7*
zKZ(>m6ET-Zktc*BGX*F}1K(B=5;SoY=VUB?LsI)8j#+>qgv3YQGCPq$Ui$!cRqR$&I)DXP!mQpitSw-L+*6)ZHfSdBI=kkXyh>i
z-iPJszrw$3=)wB5cX*ulQM(yrL$AVOmGr+3DONO%MAQ7`Mkjw}Q8S=t%DNb|G%{I=
zaP%auiK#r!j{*{HLkmHnb|Aj;P;?ZSU*0T!x^PCq#Y{{0nVQM}y%zUTx%QRzDpB_7
zzOb15eF&^PEmMD9XA`VoM;Vn1mAWj|V2;{dcEP`@@87^OL4?smWxO<9A&%*dV4jAKdg0*NP}<2UIvBy%Spf0XuUz|gQXMm8++%?V`ETc0
zS&BU)ec&2O-gpvE@*WEDVga!~^k9g8eRe6IAP`Q|P3k-?s7
zB%6R?o(eb@1+r2bVRX+{5jDVE=(8R2n3_2-KlRX&dDd&doGWJIzwwd
zi~&U?M+|Gh3(_TjmxY)SLoqhGa$`7GJPndH{Q!g`d?uD_MX`SHeaH#&@Y8f_XT$=T
z9DhB5U76Z4x{hit2k;?9fZ~r>V5%B9wN{|N8l8(XAKWTT_TIXZ{?YFu22%IqKi4P{
zSBl+Nj(@O!k6LXlx;8yc=3n8@%eBeWOC%lVa7W1NtOe?>bkii~Ck<^x)ej>}b
z<#}FMIsF`p0`=gqWOow#WBit#BEi+hd;#LW5A>*)ke2~Ujfa@h8U3-e0PgL
zyvVm@re9I$zwcdWUQ=mlVQ4_2O=
z2t6T0!|h=);RjwB{{~uz>|L4K@KF_Vz1*H0q9w#%b@XduL|BzXkGI*Ky0pWPe
ziLbfzd-@_{3!?)$F0gF;O`TcPQ}N?&G^3Ry%buS;pe1RhB836=6Iu-cYUtSDVBgY$
z<7wG8t$MM*)2}aOPsHl9%^Q#mI9FP2A_OgaB6J$)bnAY!}CY!DV$`S
zaG{tot14Pd^7lm=JW8XVB}H>XV&(f@J>Mx}O>vjf*_djTxJqc$Ah6h*uf=v*QzcY@
zr#&&1r@!lPszx~2eU`Zt%JC#cy%AJ_Aki*ru4ph?hmvc!Rzzx6q&60-0;Ltckgh~f
zcCnWD_iGBQ)~IcqEUF<`3)R4|XbKVz)`H}3zeHR(AA|i-f!lI9*FhnyZ~VVuIIxPw
zj7oc3NhRWZ8H|GGL#rg%#~m_^5vywfk`7!eF#&z^T|7ms=x}QK;q3B5X3~UJmVTPl
zl~bW-yYfS{at0hn0K9b}d3X*pem~*T%ogNYyk4h4OQZ4r_@)jo~lu9Gbwn8AFhoJRQGMh^Z|xYH{-TFC%%tNgv#>
zJMwQ3$*=NCBJ2AH6;jDv5(tOUln)#h(G~Src$CT?EOqS={E>#>fhr2FQ0NE`ZZf3a
zClJY{+71zwG}iemT5^lTCi97B@Z#7qKa7eoFiF{yl@7o!MCs(diGfBrsMI+aHYzn2
z@>Bo;{w(FM*eP_P9Q4yTA!Z?9_H4Oz{lUVp2If>)?_JR4V{LWOq*Z5~+mG<`)#t|CP5KBkA)br=Mm6m1sP)^eUcW{OG57oP8}Wm(W9ai_r8riN
zeUHzJ3+$MjsqEpkNTWzd&`$~gk#g|Pj7zwyyOkjP>gvUzP7CplKJp0=2HJE9@|c*g
zKp_WW%fzHmYV&RwHr1Lh73xw;&NL#+&ih_D^RH@aUdV2D)&`VX8qPFap$`O#dtLYr
zEnh5o`nkvmd|`1{v1P*emst0x_Xde3|EOrkpK2F~r!88RlZk0{FB^0&72g_Oh88DX
zyN+(@6FvX+;D5gP@MZ%9Jq4eFjE5})F=&7F
zhn~x3t;SE9Sq(TjePOGASs_7yR=Ykz
zvD$2H66|z_w$~Yo6prUPe=;v&Q?qgzE2k1}vMMM&OgUK&XiNf}q!~L5E!s#!*`XGH
zLhUn)x=DAS(-)pL`w{9g=_2*^OEUWkNaNyP*
zwG}0biS%tI)a;5PnW_rQF}D~gYh_2S6OJA?FscnH%32vHVzhYnanIa03lUJZG&^ci
zd*>)fd0bY3yAWH88ah(Zu-l5?;tplNwMtJxm2;tcs?Jvfg+@Y_Y@@VWKxGzH50`gqoryg
z#2u)vOo$I86(CYZHV6D{&WT&FNZx&xe5aWxPwWuB{}-<(gHG8ZByW?jPUWU99j(8^
z#o#o%Y>q`Cw@ooRbVyQXfOD
z*10?gF8;5m-IzUT^M1;{i>
zv$(_>r&N4njg}6~X%+MR@#AQ?a?O^5Q@LBtgVopRH6V800N7tOMQ*0(bO;i50fux|
z_Kmv|R4Ac@Or2UbMGO&owC9jNW-zI}wy;O$3x11U#O8KX3(l?WZ1x1gxvFw|ziMKM
zi-P{yPbT@!E&%G34+}Ad=j#QdHNte92^{)%JQi(_k-jCQ32b_7jFK~lT8R1|L6ePf
z49(vX%k>7vGY?L$w4)m^T4XeRfFZka7i27K-XxX}*m>bUe`J(gQ>fpn^@A3Pj)}F3
z3N%FHsx4(Lm@KdURiDBL*H+Ta{a85}_R}oq*|X|A`$L>pgE+x%<=APi0jq?>abH20
zX6vRbZ9SoqA&szw*~L$UT4^UGSmr35o=|JjSUVwqCEKE*i1PW7jx2>@X^P$p2v
zEnHS4iR-B&5^);KSta?E{SbU;I@(d<4kpY(eAp5BK!G(D?yQVt{OrnObY}2dkoIvD
zn#(G2^-i6-EG(8TSDj3zDDdv3?`ym~`JQE33-^>6HzSJfaJDl9>UPoTMFH5HI5cwE
zN%v*fvT3vCyG0pE<>xfR0Be4|fbjTKM^4?oJU@BdY^&oBgu0*N
zigZgu24Q($Dt?i*uf?cD5nI~5yTS+KxvIClf(98W;Sq``)kZ0So^j%l*rKA?k^9C6
ztF%ZWh7E_#YQ_=>9@&h`_;}Z`mB8@J5i5Szc*~7*1!xskQ9N5}=o8?2xd~&(3D^!2K2;kXOBDKRQ~)9yQO`RF#(MMSeJ;1
zLp|)HKn)loVlIt`uX$JQmk>&6Ly}@JtR$ngnt~y3Nhkj^Km`BLFInyvp#tHjGJ&B*
z^UtouUj+&+R`e}Iq@+*;>jL4@u71&U#?P4cpB&r3M?UZ(YR(4BhBTn^pBw-JG?Pu2
zrWPpf0}^6~{^Rrr5pJ-#Tt3(lOLM0@*p$ykjQsiCX8Rpfr^oGJ!2-g7kgDtBpc=&*
zA1Sv}9iDceaJot$-v_?=Trj<}U?xroc?37tFwV2%29we(aE`Y2S0*^q&Zx_e$F~g&
zUa`vkJ-(4a6xiz0*u1s9@?)0py&jlah$bBq^a9RoRB7CJ>={uKo#!?h1|dR?4HLu!!`N4LI^56S7_jDN={PP37o4kQBnd&B6k9x`DCOEhYJQ*JKnzo7IC
zGRN6ONEt%&7YXAp*4x%NfRdTW8h#F
zgL?q|vmh}-fyybx5;}1CnUujeV?pN~ukY4eL8smlM(>3dO7R62u|sg61c_yPF(0)Kl(J~s>Xu-
zbXcOnMJ1`cwA1Y%rcx|uO2k+l;0++DtY-+Sg|>lEJObBhT4oiPNLBjj0FuK`hm?&U
zD6GMc9)Z7qwLqc+oAf0v=Av+AT_VQK35Uz<>+>u0L%bc#ql!L(Q7Y}_YetkwCjj;Y
zpx~uRAlZl0G{atXd*Sv+&6@rQmfJ*uW0lvqM?gI^-V3NqsxNg&XUgp0Z5n+aTEdSR
zQxU%!J{|nH!-Dk|kaw$zEq&`xKc#A%9x4$y9qb)o40nh_aIGZ8)R&rCdafEPA-_X9$S@
zzBglo9FOlc>Kg@)
z|J5>g-uw+yLDAKm7P=DZmNn(wXTP2wXMU4y6E=X94DB3Z@DGx`XnoLmut}kIf2#h4
z3g5ndpV|k<%XH@^lrJj;lvXP-&de>q+^6m^AYqNocuT+nx&N^g4tV?sh^;6v5b)j%
zbi2~i@lG2~6Y{bo7G!a|!H8552AAFR+L|F~pDy+riJEB{(L%1;4O^J>xOon)hun7_
z4#EoDp1qSHN!!UiMYMg>ja}Y*Ky?L17z&)0B@v$H_crhsY-rPNLi4brg;G>ZM#n8w
zx`#UDC=IJ^sS56o0J^xN53Mq*K2CXg=ac?=)U0~iiI1+7XA|Wb`R+zFy%l^9LH%*V
z-S6L6ITRb9z(&O2SYth_O^9Rc*ko8-Z8?Z}gvAi~M0RalZ>@Ej|6?2G$3tI@mXJ+Yv%4w-CVM
z^UWzHYL>M{Akq$R5dQ)-q~FpP#HKJOU*7wfaNk*P$Ap+?kc#xTjrnpzkJ2m(et_Wz
zI5rE-h5+KOJkQuieyo+4@vJx(vkqH7go2eTB(|`36Gsj{3w*JKLX~;xK0L38m<^kb
z&4)`_1w1dR0nT+YB}T@CNL1cEDwI(_Jr+a}!>}hcIXlROce?q{po{|dyd{m$iEO*F
zt^Sz2Y_T=pt_~V5s`DAn9-4~w>I*Dn?IHp_+jn!Yfz()x-`PaMLKq+YeEFhN6#IC+
z!3>H+X1p?Ij4TwV2yoqCI5?>aa3_apYgu2L(N)>s0hyhus4a-ZsEeD!iM*4F;4~a&
z3Wuv8d@o21r!?=dMPlx+f|{xX-5C0UD4d?34ZZfM+EyAQJ+;~#*2DH~N?$M?-!U`b
z=-A|cLXGB16U&e%7$x_=nIJ>|SJ3AEr^qrP{kKd1Z<`S;7{Zmn0L)FSFe*RGd82Xv4P^sxbQ<>X6CMP!#)A{MfH6n>vn@vxpb}NMo!Z
zH;P?ZC&B+Oq9D$3t9hwEptuxGNC?s;jFrNaBF{5L)>h)tgicx
z9Af1!Z@#4(99%f2iBB^Z{Fvv`7Cj7h^Nu2lhD!=>W^d~3;`E;p`Jd>Qi8VZH8mm17
zCWx1fJBs7J6!v}k
zu_8I;M2^HXp`F{8>O;DY%wnlF%!UNm8wCLx9~%cMC447WvbJSKt)UjV9BJ3i0Bu82
z5!~6WX7zpZX{EV0ZDOnxu@rmc5_6rM!+n6%TD_L=oX6ITlIG*W%2oeit%+7~x1A1&
zjrc}Z2~a$2LlNF8dE1^#+JVs5O2Aq+1i@pW%@p28Zz^}d*7o>2*u04eVe9lbB6w&1
zTZz*F>;Sk)O?y{IGS|t+c^JPI+^c)H?qbsxNTwlT%V8_!?Bl~zASzwEhL@x^mJ?SF;xnl
znq`Q5Hv|yXLmpwn-{Fyw3UK0;F%=sO0!R&UQgu^y5K`fctZCDyIBzsww5>bO!-b2T
z0BD*VrVn`aOsZXs@3b^LtI&7(U>-~{s3b5~fxkcOgXoY$?G(or2R
zpCcw4S-^&H-M8}ACtjH3jBRhxh(n3E
z1Uk`Da2TmnXPN^Z2+I5pvh&F2;Zt?K0hq%{*E35jxQ>9iiRik@3Zef-nV
z$bBzF$L?mlt&w(?n4g16UJ{bk>GkmE(z;g|ei7+u+YFfIb;LuT_
zGiC{9Itlh^$-EhN(lN@l7#JsZII?i=JV?jb5<)@@Bt`!awxJ;pWtaGNoq=_k0w*RM
zW4NkV%eL_Iwj_3kKf$fgT>ca!urmr8J8_-uu^kX>{E_XXTDt|G8gp`jpy2_Rx&+i)
z=l^{uH!~6yU#=gMz58$enG!9`47mmwkmedlIzV9AfajB_6kl+IA_Y+}UKsNCj{?Z@
zKp2uD-%C4UmLa?$SfZ)A;$(!)fc|jM7i}C};vYKfAC)xeA
zr~vCp{Ev@lV)7c@)Bw}im07Rhsr*|&as}K*iEBL8XZ#eLmrg9!+=}ZEhsNo-&BNRE
z=7o$$%?fdT`=2UfdI+{=R#aTi%@E$_I~;@+RJ@miDmMwZh)(Fjauf^~lmnEY%aflf+aJ_b~pXCt?Y_cNxM+yEA_4rt3hK4wlQ$
zMUE&6U}WRpFhP&AgwSS7+MY8v?K>%DM{W%Y^M5{(c!rM%8yad_cSI-1Vk@`nznL
zsTZ*ioibPmXx_cY#@dZ8r|R78|9VTE3tWrbZYjQ`nYcsOnCv!CoiCnTp#hk=I@%Gf
zv}Kf75ei4Dndo|U+x655p%Lp?R10O2u_d=s5B~0RByICkvm)|+np`(EC>`oAhRr~k
zHJ4^W3|B$L#I1$^vwhjnvC=yisGs+Y4&q!0UB28LVHlpt&gSXPL?CIgbH?Xw`<~Gz
z1u$2h7gvGXBwU8Uo9G}g<;tLi>bT+s0HxWJZ-Slu7fJQimCpWMbA`
zqdvSuMc6B91fxc5Mv{Y|8ZZs>j#so`*Jg)~Umu^7-KEgLqAZ&xeSkf#nryxMPe~6r
z@E95#btVQr4hFP>
zwOi6la7&%WQwm-RUc)1}*hERe#j&>9SNZ;eJmhQDR55@9PK|cD;`WJfQ6N`FNC~a{
z#X8tLX`h61HNE>+0_w>Tg~Th6iF#u3-|EIE0vO3--
z7AQn_pRtFAd54Fqfcj}r_j9}W6Jhty{o>(
zgvdhU+J5P-e#$wTh}n7>_SShN5_Y-~7bY;Y?K@^>^ab0K%|%A6rO(FAhyw;=g_9g>
zx{CX?o;sZs=q7kJoeNezcgAH0YobXYneQ1$OmFAH)E~-E(*VXOH}AuAQTA-RlSs2+
ziWKG~(FO)SO_|5~-~^MFwMFB%i@&p#i
z<4#k~Hrb2SfhudK!TaLu>VRsnseFW*2pbi7C6t-F+qq)+%e&IccTEMElv-YynY=M$
zl=n=?)gnb_wkssiG!Id9ZL6!g(sGfsrXAhAfN2cnOPt+Vi=R-vOrZRUYssp-e#8fWdQYL_BR=KwMJ+_w?DDHKikK;9{v
zzGUlj=h~~^oaqh<{(y7P0JZ#zy(qAGSm>n^`2F&wvt;H1JunLffJSu5ao@!>z};5D
z?mpSuw=gb)<{>$D_-lhi5gKjUzva^EIk~B*;~7SknDya?+4(z6dX_Le{8V~|S~2z#
zk04SQseRg;2emT}WaasPJ9ZM5{}+<*@^Jj`d~uVjt6K8KBiBX@$0*p`M$8CDp&2$u59huc|Mk3<77x(cAvHnL
zijcDWFkm8L_0MeQLp3i&1n~v_@$lqn5^cVW710D^#wJEV!6GlLqKAbAiIX~s-~cj+
zz4VX`pkx1RV-jk)czQ4*vdAG=B}h?`YKI=q-Z}hyz~Q=8RBy8J8RNPa)2!emia+4R6L^HV?z^ON9+U}9C^bb6#)$ZBBWq)@q{z8&|ppA$AO4oUI5?iKRz%?I|V^TLJcMlV?uWei?Xwx*3r}BQ(7mm
z{UT1618@>wWdGl}v6lmYJ}TtYRw^jJ0O3jaJ^)6jsB+>=_6*N@?9NG80OXIo4esO$
z5^nr};ilqY?RjUo4{<
zxATE*Z&i~>&s_DjidXII1+NBt(p}dQ`s1y@Lpp*4DQHE$ocQ{r;S>YcjK4^si6PGm
zO7;wQz_?M5d;YC}Z}=TmSXq^WXZst$B0_?TRc!~bko?_1GP+rV{_?NN_#%Ua7;W!p
z7rIyaYMFQtw)<4jhlU5(0CZ_XApc018#=hP+54hLn$181?linQxdfo`W)8F
z=<2bSon^m8L0?B)Thu^0O%O-`!B*H|7$^RV|FHvh>RLoFT$6=u^N#H
zBZ8|_V1wI$7fv1lju17@!eZ7N0J{zj@eus{$y|ekV`6t#3uWUDSlA1~t_%?tF+9}}
zw8EXmO=3ZUuE9i@z~a+V2%p5yw#GeK?u62J$ZwupL-|0q{;~EXKrIsNXUM@B^gS53
zQS~LsfAGrxiEn%Z531_ZU}-(xfq^-5cQp$~!XSA~!=2jSss_daLgEnoa>^9(D#lhWW0gNv|>+`DblO69E((V3Jc=zku6U2RQmL%x(
z6w>8A4CTB8>=8oSCcs$34M2KfelsjCB5jT(0i&(~!$#>r-7+NV3Y$NKP2+sGh7i`X
zNAW8Fp0^{Loc#UQ*TEnJdh|)2O+WjHx?^E3!R)m`z5t=G=i?V?$yVPjDt#FzjtRmx
zItB!ozJ9=_mViUZH5@#IkLON*3$*7)Dyn}L0z^okD%jK|3*e8mfO%#-DJBMHp8+#(
zD1))T%Z
z{P)!9ivVyf=m8M$;c>pz0W12$^1LIf^
zvG%V2z6NIxE?oJQ1XcV}Tn&d1mWGa-fJl-!b>sAibzbbVs4R?
zEV?m-c3qnmGv1wViy!Vx)4M?)K&W`!PG)$#=sMIjPs0UKaqlbIsN4dBjK48!(n8Wk
zgSzK?)aF@ds#m{|T|&w$oybyYsX;05N&|Ka
z^feP&C_%ANWjUNKE=ab7g9$txp(ZP%yDNfWHO!%HSFzoO)pk4MDjPjPAu!pf)YbM;
z)Y88Zi3vV)N)J|#r^7fe_|R_KYTgwXO!kYs3n^$Yc^S&wyH+>
z*_Xybmfh-V;0--!bu{z&Fj-Ps-wue9LM@3y?ik5zG&)t7AiRz@=MTq*NXy}0eaD|=
z@tB{Yd82G8&|^^=Lreq5hI`!BegGI0QtW$WV&S)*JK=Et;~P^_XKhp2Az`cdaD@}p
zFxq^C1%N2FU9u`ccT`^OFOr%>EzYl-Uf)*hBzFEmbGc@kBoSS!x~kbh_gdZ>&df2y
ztz+7F^stB--$slUr63IQxE=NhFekxM^P~>+zr1G8s~3*HVq@!7M#Py@Cjc?*Oq%6<
ztZ`(>w9_>(@585j@y^2}i1>*a8bvFRFwGVn@4L8@5CIy>A$2}G%uz)MpCn{L=j?gM(o?@eb)Qa3fw#&V#YQxy;m$fH`u)Oiz
zAybiF9BZyF<)y6ahbHOD54#OGWY_m7hx8iuFAAZdy_F)Stxrm_~2e#R_Vi=hHtME%*t
zhGUbON}_-gk_xE)5o^y@yC-^)#KUyFMmBxCFkE~f68&@VXI4u^76q?PNnoVh&Ee~?
zWa?-Slp*tN-vau9>=pQJ2ENN9XQ`%TyJ(819l4S~-m_x~yJV|ON^}Eqs|krFf=rw_
znxYLY`Je)u#+GDnTi#4LZ8^4ukTgjXz(_&WA&cSVZTn%YB^xm??^%y}kPqmEY5p|V
zKJ3Or-&5#1I_lu{|E;3^K3H1*}ismjY3}>72ZPw)@}inf=pk
zm5U!?9eN*{{8^PYb9;7we&p$gJ^OG)#|wL|nh73~
zZU!$}ljh>)^`Y9H?z7A|7HntM^@8S51a{K&6<9W39z&iAnx`(JU0M^wHhrRx6Th%mQM&<$B#7)zdR(GD#>HX?k*r
zDCHC{ID5fpl8O}lt#kD!(n#57(%ud(^!577;@eJ1;W=Tc!&5Y!w%p=D%Vt
zHZKZsI*4caF^>LuL`TL_(mSuQvfo;
zUSQ-@?tQ#Z1ygKM>nZA3xS}%Wu;se#nMfaJurM>;Y&Ezz$IboGFiRY#*^s>
z6Vpi8e7VMF7WiYCPO^NK$~HvSz#M|{R<1>uB0P7k!!&8s_Vu`p=5*X#csMD`L!egO
zY_vv4y#2LG5T+Fq3!2~k?hd##Q5s&zZeM#?$D71u=L=<&;+i5R4{z^ikVLw?1KcZs
zGFSC3yC1$~LrW^rTQH-Xk=d)m8S|=`j4;>PcQbHkdm6J6cknD33k8|7Pba6*P%~1c
z>L`p0`tsSUug0GEJ9`Jkn?-({=JW4h5>ijqviRFg1h2ZQfA|u0Z4d$uYmYozyms!K
zJJ;1S^Q)3}P5kZ@#dl}Tc655wfsg8@jJlPVyy;PK*SULM6uGS|-Yf%+)z?u^X(AaX
zTw%&L*|^<_!~;=RxHMPC=@02|xwj-oc&c&v-o4^sCN00rwJGJ*3M}HGI!GT}U8n+S
zQ!S@PYcRW5u6kCOI_*B!rgL+0wLCn)LV^(`WXIW_$ll|9#u`eiLMl?S`CvJR2s
ztlHe1j~3P-Q{g6Lvi$}{C~w*oPvqE52X7mM$#20Iu}l%NPJ^$90~1otz~p7Iz)BIS
zG~}a5gD!nrV72qXE4Mqhlcxo(=^dE4_PO
z1%Euxz>^|%v3`C3uATj?R#Q;!$X>Zw*vRyBn;&NJ#u+h6L6=B-4p;;a>`kdxZy1`x
zGW>0kQzDwd_iBpI8N@qvwE)hXE|gD
zL3#O@Y)AT?j~A0ek;ND5GC$Zz?Db4Hx(C&(2*YLvb@kgag?Kli6y&6;mUc7sWNUwF
z{OVJymAXev7jG&NVrh~pT6kdy>3k)*K+FPrMmc0K2NYDWdD5+w=vyWp;(hyJQ0jI_&5@Hp8?40FpA(?h!lSEQO^?ee6$FouHPo~;GXIS*Q7d|L9e>-ZlRLxk-ZcjZkTlAdwi$WbM
zw^B-o0)Kio)$1&6XqIv<!QFPrwF(`$KE4yG!*aJr9JO-qcu#<<^
zxzq>KbHcWhu)-8)9F2qtywBgg1ZRkIuvknYk@|uC+krx|7ifpRCsr16299
zPQYq^2DNV~!Db;z9ro$e-XNItbFcn_BlH)g$1h!FogH`r-Yp5rk{DJjc2#KkIRg
zvg;MJ!4K-_juk+a8?0Oc5O16=;CW
z8K@qffxIiInK$Cf>HMH;d`n?Dj8M%XD)P&SUVb8G=blX^)u@m_#-t^xx0;I4aG+i#
zXHBq06<(_|vC8z4bk=RnkT8G#G?Q$Z&@nEkFioV0Ki}?T5l%Bef4DdYk?`2WO}5>!
zoG)M4*42-#8WuOgzd87oHARG&1tblrvmOcM*Aicq+=nc!r*Z}xV@OZ(Wlf1a72T_7
zfBd|jZQMw~6F5o~@6v}iM1*=9K02G$4#*nyu;!zGX8~xs3C-l#Z#3^2C?%n!SLt91
z*{dyl#sV`y--!5259+d_9&~6UBkk#L1gC>vIPI=*GnbkI>0(=M+n)p>fKe3)tii_8
zfy+?!Qw`;SRiing5DwWw)pquW6;9Xy@|4JEO7vRrh`%%W<@celC0rcKx+L%(TB4s?
zRN2H0HVqRao8k(W!k5NhFh`yE;=RUug^De@o5|@Uz~Vb
zsCZ1UPjDa!AFkGb{N8RGw(d$qJ>=*vxzO}!q{&5H6P@}4KQ287Y
zBD+GnN)E%kim$s
zh|UMdKvrbpPXrpW*F>c1dDZdNj#oB(;khdBpti2p`jnVc6s5&JFGPS=DXSID#r%kw
zex?35Xg((Vb{(~ut;Vm6;r^}l41F8-2745BbfRyGM5WEckWeiR6{4K2i4LZo^OVBDUFlxaPWUb*!iv@#GSbv=
zW#)5qQjW3G1HMdY0FJk<#>-%r(;Ttwa^`bq<9Xr$Qpt#pGiK%9(;3)Xm1kCmq6BrK
zDOFS=9C4Xz5VGRe2@6>|rHv@p!Cbu1f`GX5cejH>iGwE4jfRE^`qWXKe^^?fC$)Fz^ZEoo9mPnfKx9|KvxZ3XdW3{a1f;017TauduhW1`^VdGe`3v<85Y
zVKU6JdZcRB2olC;WyZ)&2|5_YF7O#e7Jfl>W}|&eYu(+ET2;?aPpSIjYI-6v#$cMTeVlT2nva5ef2^4B93N3
zz)yufkGLe?Sy?ERh=Y**ywgzOwyY^T7ZMwVtmO{xYz?%^RaGmuS6od
zP2oIS;r|oC7h8wXICI4;63=U<_Eeq+=&YQdBd>n2gpfwSuB}U;smA1x6S;S2WkqW0
zUJ9|dB1(SBultSjbb=FHmv&?pK}S?O|I)P7VF)!UV%ze#sz9ZkrV|45HeL*aBRAPH
zDnAV|EA%bvwEeH*`3^=5WVZ(dj52xQd{<_bSKK^|iZmVD?-|?B_Yn~SLwU+5;4}2s
z?;@*DXgO5T413jHbpy)WVz{E-txYg79@z#WH!NQzH(#ER$2diIP>1ed#XRW*X0Mig
zA!7zkN-K&J&dMI;Accp2$(W+0?*2z*>KiP>;QEQ7wtOmIskc845j&&;4S$wC4v!<
zhE!q5Y?k8fe-VB0O}L<*WvZ+ar?!PIw5yGT6^tw18%o&ac<3rm(}8q;ub;e-4I4s-
z04u;?8NdoSc2fjT%UR=tBTIFPrW3kAD*3%X!DGf%E}eLcrP=7sPc-3&D}YtO;g
z+l`-7$P<-6-J^Z!Wtflw1_{{K0_rUAgTCERU`wdx{&LOH~zN
zwV?z3w4R?fREyvKHDTqSj{YRn>ytT}{)*aG#10V`8BJf~)B0RGD
zi26cO?7~m#tdc8sj(HA){qfYvw*lUuN#-2Od+)lA9dkWko4hmwEKGjq!VgXuPUwEm
zSJnH0{3ULDTd>R>9vP`RzoN&}dd8nCWziR1Hs?@xUDkayt|yv>UOiA3No}g7yjhk}
z(Z;5q{F@3-(!V^nY<{at42QUY!CAhwcDBT1>&ty_d>)V%eRdT_{L1}5FBW2s+YOjq0h`D
z&e4s^n>e!^rJ_L{t0>U~z76)u9-#%l$ORaHV>~MVoyThcLb=|V#xlA+$XUN7i8vPT#qiYePYVx0E7dfMTk_zYR#Bgo?~5rQvjPM?qU@;jp?E!x5%8D4GG)Wb)uevRB=F5xNeGA@K~`_0
z>rx*8`xlW%DtlNPOtTl%dyktXOqD54YkJW7<$;)EvnPe3LMa?(heX-y-X+W2Q&U~5
z+$tgzS++WDYo(aj!+%d|sozne%NtX}|NhG#RD_5bq!_mjeI}y$P0_hz@M|84Gw64m
zawn78Wl+XYdMXCBL-YLD7>Ary2Qa+vT}s4&$rDk&M;zq(pK}Y8P?gto-*J93x)Dr(
z7Kt+Wc)C6krPI>iO&SJ`kT&qAeW>acx-`6zoxK0eTd
zkI4>JeHp<6VOl^7JkJKxM*iFB8)t-3_+dJLQ8S?MpXb?*r}KKBh!=sV$mm9He~orn
zT)Xexz{L->%F{odKvLeNocCJ3t_EVFt&F8ig}-_%`7YV=K{0U~QBi)0IPfI_d|39<
z0%AVHfw0U$)op{hd)ho-r2?{9l*ghjtZlK^?d3^7MLqs$0~sQhVut_>+@9{o*^?!u
z12Np%rRQXtzS(S6o%E0>e|kt$F7wkDY;vs;Z?Yl6n!W6W_*aGcE6F=l)ag_(JPs{|
zNNge}^g6SgDj3@MG3W;CxbO;q%(jtPQr42wB?23>d^}9QsV40hzZ1`6~+k-W)4g+TY=Yyb5>jmgD-*H=+AdBVJV?AbGiuQPo2;od_2d;PND^|ew
z#Oyow;gT`Ea?`_U;&4ehCO@gInj87)T;s5?wcCc=NQCiDBE%{n`)63b|=~2{+Y07Xp1pTk7T!oz^_0!$g5A+xPgh0b`(6Q?3BH4
z)aqruJEhq`Sf@?%;aM$#jFz=J#9}|;aA`nV>g}V
z+%0Knw^cFfKuBV2?N^?O^zusO;!!R+8YP-WRh1>m5)Ik|LLC(=rI_@$24M@7ZAo61
z1=T;cUa`!7$$s!Q?ea8rd7y+mZ>H8zrjkh$ZmvhNh_%c&yC{0QCk(qS_)hb_xR0*X
z^Ao~D*HNU%h70uu59l3ujOM?EAK($ZI+mtqQn^&VD&HVp8{-*V)!~i`PK!!Dt*DkeF?(8>j2THPG!S346q{
zLrarJTp2m9-@EbQ+_gbO-xU+Op&D{vNV^#FP*wndg^xrzD<*|=fcb+}Zs|j;jsiHX
zoQ;Wt^WB66v~y9dlt^cjyH=9~l1`<})le252^P7MNDbk@nM
zLRf8qi7@yuu&&am6n4a_{`p9*zc2_CfSri1sh0wO6=>m&u6
z-mmNXuG-Dim&LJFv#P!`cbzRc78y&Bl`6?zJH{%?0=1v>YNU-%w)GgX+}{j!Hd{Hz
zZ*)CraE*UEDaRO%5_~J#u{Xh|#C|N%v*1((+V-&JUoF|-PC5XwCE}3MgkACkxhYSP
zi0Rhxf=!R({l0dW4KJDNShjYM?LfiaY-Kj`-|TLpihcJ?0xEdIl@8O@*|kGn;~XUK
zza$dV61))$z9z%qEQEau&svur?=)tWZX=j(m!oTwn|7cKY3?p~Wcf}7L
z)+I+jP_zS8i!~OA1$=ioclSYDD4^mJDQ=3xke{4<^ObGPvs7U0&Gp4Z`bt(Oq0Z6M
zO2V&r>ddt(=pUT1VH!&P&)Y;{WcmOn7I2^(|6%7S{y}j68+>CTU}0nb{}OSm9L&kSssF?v4XSQD
z3D#>iu{5zpdbV#!%n+KwNrY=LI=uSh>*EY2gX7C_IwMG$wK9RVM)E?9KS@_a9|Nqc
zDMZ6Og|F`B=bpR)@2-~#`;0R?{3SPCVnPFR+Ujt!X1^dX9#K39#c7c7@rhWJ2uASG
zzC4wKN&y#qoH(#|Yu2k9C@0;&YGTZICn8;e9qfL=WYY+7hOl>~$s
z0Y1*mn`IjdB*h=4Ko$fs0CaX(=eHz*^1e>2Q5trCo8e6k46SIk01Sct24g}dmelPn
zccOFc1Az}D(t+ePv4;UD2_9^Uz&Ig@-?lij0jT*myg%TK5*gS6f4*A*0zbMJv22WC
zF*G%owY0_r%mORDLPWlu@N3^!T3bc#gM8NsfW-m1*asp0WtpBSkBuO|cSZMuz{>11
zldrxz)6t*^h+Dx%dKqoy2_HM{nM&)P}@w$>zHNmT)ctYmNdx
zKYvIYeCjvCm~}#tbvk}*0453M8Brn&}>tcP_`qNOjxgoM0)B5rH_7E(M%k}3#b}e>?viwrF
z%y$JfWzgSJRNa#qy*FlFe#z*Otf}VWeBWtg$)S8vKt8VSE<(wlcfNnY7h|JEJFjtm
zuZKsp2p@8P^BIKr0wi-&Q@mH!1)*|4d^SOjqk9Ehu&yR2qA(i`iy)CuBuEfI*-4^c
zAoMo`R?(FSFi)Vrp~S&spggzjZG}J68zUu6Nd#3xI9Rvw_K;gCP^wpGM7CIdId20s<
zSn=nsC`1H{Lq~cFhL>jRfAu$$rU4Ipp}diQo04jEb;0>L+?PNIB}Em+&)dkI9+V%+
za-r3Gscps5)tk+?TA#;rLy!tXL3!0GkVXf@6ypxUIf1yZ{?s@2=Xl&AH)ltS5=WfY
zXy?vx)dm*;;v|Ma$iP=!De(B=zmd|@2;6iDYMJiRmq^^bUiAXyCM^!R=QbfgFFod$!`!8TB~<7(OG+eV^D@+Uy`dU;QbHW(dDYG0zfLzAEW6@Dx`c`vh?9dqE~Q`b!dRP&8EgW)Bf~
zh@s2_Be3O=9dxPE4BUlwbkABCd|QGs3ovAb+#s-a0L3XR+2+JYV6vrBuDI&T9?*_2c~(LLZ((
zg2VX>k*zcMo!AThNvV;Y18Fygwl%-@J6#h!VCR%JjhvnyHEFs?YcYl3=RuP!v?T}G
z@~qVT&o8*1-^v3UoO2OQlUz`Dto$p;O(rg<4{}2Ps)UaeZVwil)Et#}crlP$#_X3(Bif9~c1dvM6M
z-F0Qkrv7OyU3((EIP3c1ab(c6?4JP9IwI9bh{O$=&k@r#M@XPN(Anuwp^RZlqs6<`
zBzyH}eYE;DmzD`!rvmf2$^R(|w?cK!9@29hEp55SwaNY=bvmtRnW^UhC0lNxIH1Y%
z2&_d&LKwY@s688tgP+EFMd{US-r6Ee5PUASbFjr6TClv|ebZbzx`TSFTCxM+=n4En
ztQ*C6y*-OoS;~1=oO}v(G^pmcs!cnbM;nCFI!si(OaJFPIEjkJCqN*E=@oLxe9;u&
z{`GIZ!9R)B3EbTj*cAmop-gTTzbzXleM_Q{t2^G(@iH0#!@JJQ(D73s_M`lyN8;OE
zy`!s2X7dk}|y6jMS>0AO}uV;%%!FlD-oZiy?_}eF{d;EB_e?`97inw{y_mzFq
zOnKZCr4U<%=XC&w&AQd=XQ0F%%=l9b8YapGES)_hp#}VBA!8$|>gk((7mawPNxt&v
zmicv#B>OY$02WLnsD<)0gS@4y+d9{~>-~!`=De6Dzi}Z3AnMw!oInjIf@WvyU!QJw
zM=JA2yzGha<5l~1kr)U-KX_-AS)p5bWWe5${eiB;Z)=7{*)c-*RzuzFwtZa^IkG!D
z9yX;m4OeEzc(LN2Ss^`h%xQZ*(IkZrVy}t~s224glgN}t#0F%G)XVsDPMV`RT@7ep
zKft##a6U^14UZj%YC862Fe!UQnJL8gSk&H5$JpQCs{*6tmN@}Sh@*l0jB`dGvX}8V
zS1B>`!i8YY<@9sP#R(|ztnkpscpM}dGDS^7{{{7C)hg}<5xdq#J#+q;*}CgZZ7;{@
zIX~YX&!x@qOcg&~!>ggcV{eb-wngku$2ZxizR;QYIb$D#?qh|vPQz`w;V8`HW(jDk
zoAk+svpaKx!)Jg?&7O>f+<-M>uYep;MqbP-;kFRTmQl9-yOeKd=T}RC=@#C)6p!P5
z!T_<{(e7PTRdIL8GWMLzS`Q)d3An%bG)&!LqcZ!u%CU}A!j!^?cITaz^IW;)7NWgU
zRcI3>M{_|UiS;I;h?!`Y1h=xC+9!|Q_mu(difzF!(P)6k#V$tpw4-1N@!&WKPwG#}
z8s~4}aV>lf$2#MnHxmI({P=e@{Xi7Ah27xTK2(>Qp|=jlq5BpY>D8^|?QrtB4qBT0
z$yn*$)m`i;ossdJFKt%z+UYEPR;#Q=UGgQA(et;=gTLbrAMH{em|g;!-@16nk+Nc7
zW8)qRm!tv3zuJz2J#mHwS^PDiq>kQZPaP(`vZToj)SJ;~=1-+;&}`g{D=LIHkzHr@
zE&CvsxidR?av+`h!zV54lM5Y@Sry%?Q>s
zFJ#{B?MBZ7_mehfa8a}wFNyI08SAl3jmMslNCpD`k+^WsXR;uAWy`c?sd4{m9g~lo
ziYx#%RG+~GceNK^E+6^La@{E`q{*-jB
zor5q(1scJ1gAwT76DB9IojHG}h&!TkG!mfXp+wHT#ATZx!KG~S4t0{vVsvwM-qrba
zfw%+#(d6pXDXERg`588Eo*-?sw^v3`RJ>#oR2prmt0S|+#J@sXi&cwdO5jWmU?HZf
z@3~k-!LIhP?H*xxGPiJ1OMa;6jP0kq#tEN98heYH7W@n^E@qS35y&%iV3{SX{8tEI
zs%+Yw!!r>6#%HDH7CuA!EPlzNdK%%pBOJL?t(%u5C!bDk8IT=>UcY-}mz_PI?BpX|
zV-rF4n!U4AI$MEwnmna-U3Qoe12-;4_cv2XJ{$@1+zAWHf$=3=+*rXG&2HW)yZo`S
z2#eAlk;`~_P!n{G>+Hwc*l5fU(=mWpS359Wf|Gjf3~!q3sB?SMzE5msl(r~Nyqg;>
zyL77L4d%7rW#CiHMdVo-#iMC`$H6sG$C^=0$N5j!M~d3`ns-|6)0lD+_HyH0xgHv`
zIBG&H>AcZrJ1jd+Mtn>CnadXF=lfkV*1UixDBPQFs9YQiTZz^UYwxyA^8}Em3ybvZ
zAkj!I8{!_GnvuPp2?i8=r0VQ9)r-i4_%UA!(v#r^d!9ZY-i3dK3h$Mw!Pwx&dFR-!
zk}Jt*4~-oxtf4J&BwU_RBWC0SCj{T}T{~4mN;X^D6hxdWqq-y$ui!R0*z$0jF
z8e<*85y1nz@#7w`(g?zyvr~c7@`Y87uO2QuXS6`x!FNq;>RYW}FnfZI{XSk})Zka{
zkVT_%y!1kQqudDs?R5?=tV{n)sBhjpQj%fa8o&E*&kOYfZX9?EmPGAvFj`*Ud%zCvUVlEA8hyT
zHw5R#1tY@LCZpYLGODg0s$XAa>UMFJjW>B}?`a9J1iJ6w*hm1je^hd#=d((T-YR{o
z#Yj9rwUaZ3cJ8t7h$ADax6Hw@Crv~OfRjaDh~ux}5$@%0PeAfRMaDRvC$oWAyHfx8
zM{Z%CRLaHc`>Qtxx967GF40D4%djA#eoagOe()ppl`Lhf-ef9#_BK*<8%=EZAN6!)
zo{>^rE0$$PJ8nIh36@3T$*2>7B5PsEy(3lrD1*K#iMf@KV)U-*2Anx_II~tPH6m=D
zq`_K~8TT#t8-NG#QlYcBEv~?lpTF~?;^@&VFE?H%-AqV_&Ob?au!eU8
z@}oj8@2NzdMJ83zKc441nEP~`i<<~N4m>m&O4(#Rh4^KnBC40cS+D#K)=Yv)L>xo|
z9x6j-QMB~>aXjI`xv_q%s45X^Bt9DsX9sB~l2-NvH=r?zcXW@W^n5HJmC<#+Q6jW8
zQx21CXqQ)WIjF|l$mE&ktNz9@OlV&&?%(Ccuh=(FvOQ~=KoF*UgA6y$Xi5~lHf0OV
zKa4XqwbbcwL;WqOuG8nlptM5y?EKQ};oDNvlvo`+N?N1){x4spp#I-uy$+*?DAu%|
zY5`lg%z(nv`=XY4?TrvC)7ojB`7#^!%lx*oGB4XuqUg6xSA!CSw1!&VF`TooXN=df
zm-?6>y|*TEbQXirV*W*{!4c1F$d3YtsohvQuXHvpmx*L8hQTq-O1v!>k$R@VU$?z7
z5f-j(JZq_eqGt0o7kE1K9=I^>o@AdHz}IRE7LeZ`$QwP~LULhn?m0c%=Va+;rsr$Y
zGpHNEi0<-R*RH%06dkL#>?Ctb*Oby=RnkV`uFTii+Om}Y)o%!!V@ikp@0cKi+R`yL
zT&Rf64>qpI&B}%hDNfB*kJ$%nS@CffJv7u!OZc*Vsoek~*8ooZbmwSIPS|WX$adQ|
zOu!*$xS!vgWg$mscV@dxP0UWn`fj4n<~-x5da;`esz@5AYjWkq+q^7x$c~?+QITGT
zv_3a80X>+61iW}LhDCXw8FOvm@P#E#%ra`Xl*`bWj$ont}U
z1+J-B)X>rChR)upjcq@lj<@Jbk=0Ae_wyyLS2FLuT*aeMR6=3VN%od^x@g}WEWqI^
zHIeVdw*dWpVqnrsBvIXTlRoVPRRn760JKDGWyMEx>_P??H!>oh7nXG9%xjRV-=kmw
z6tb{`r(lLS`;P7VdUyfxa%DV`32&sHT!sr8pMvG##6?_YrT7cU-1hr~;`(?2vG1p5
zx?5hD(uv0ggEzt2_1g`7QlKqbAs{vw;2LdKIs9s|;TkNjwX&t3bZ|pp(QFaVP54Jv
zK`!3uG%}i|Z$jo2xwpNiRJbgikCkg}JTm(diC;q{zS5&bDYZ1LcWkD&f$GED+GDe+
zfvVA)C#iexW9qr=j;=j9xi$+PtWBuJ1s;=kD)~u~)
zV3{-HyDb^q$-1A#IJptHdNq#DOB(g7iEqygFZRza{#%)pb*KdQd%<+`Cp67ifd^Nk
zT`ak6b4#p;wv>+pMYwd^2(amM1{Lt}b{1Cq4%_A>d%O97rb8%Bi5-Hh=MZK<`>PDG
zN;NxS{B9!&M?kvEEFO)oK8rN&fdRJh{d81+#GM}8OU)%&K
zo$SY!l(0GUVP8?})|fUnfLGt_Mt3rhHYV%p;4tBDr20Hf`flWin5WS&7wh3AfHKaB
z9m~6aTe@su;40(B02qT-H>)9QA!pm>GGa>UpxpSXoYeja@LG2cpNLnu!jE^Oty>=A
zp4xuA?(Ge4I82%N{VA-?I9iwOQHS%r*M2!%tDKn&KxwwG$2rs-Jl=HZz;=?4OwKX6
z1U+@L77NoG>byU=ZpIwsbcfh;?rJ;7bnJq;e=Pg0lwJ`dguRl8maDXX?&1Cfm8f6(l?K)|qXtmGiZx&IaZ={T-xZeU1EKLHhHhJ;w00fuP
zcZEWn9pfrgXH1RbGFr=+OMaF2s^#W$C1RA#{=5f`9Om*guSUUQd7pnx3mio^UrMA!
zHiwW-R|#FEe`RRn^z;j_kQ~u@5PIeOA(KE`_7@VpQD8cTc|1_D`3~cx9EZ9p?E#}b
z&8aW)*T!E<^Q2ZI2ng!CCn_~z*WanR*bWc&(%A+Ey}lFj?svs-koV&(bhS$g)dwN`
zX)-i-?dyyt=gBiIY9!doV`#aov)x&`j}?v7#1C3(FGmRicCE<%mtWs$5L^=eZO2)L%LC-5WDPwz5l16ZZHf%Z~EW!P5x)4CeWlGW^DgD>;I{@dyoGxNBr*<
z(3(Cse(XX2Pk7(-c=96_67K&5`2X^I29Ez4KSBxo`lXC5S?hhIKEG#UrQy_-`Nt}t
z2KI4z8SmN&*BfCPA`=mb7!rO{uj|FQp}>YvaO{tq65P9hqK+-XX1od_by3@t1#Rggr}MX;EnOf)8#dOIouiCEr{?|)2%
z{WXWsS==XG=^5!JJ>;rIXt@7(2-{k{tKv^rjYn*u>hkjDUE@!CFhy-)fN+sLM3diRh@{u07AVKwQHO
zjl3X4XhV$OBQtD8OvibC0L5)ONst{9O|pZgw{c~^rWt>e_=gO6oov*D>N}-9Wm4F@
zjp^c#!&bwes(0`mU+QiZRd(>5;vt!^8&rCfU~wV7VYm-tZiGahy=1fjh}^p?rB(f#
zkJ|URp|Uv=7ZT;F@W#^Q22!8AONw?X4%3$EyD=VYEm8eO{B59x>-(^E*_wAQlDT?}vB?ANDV
zr=6yI66%c>>nX~d+HUUxMnT%P?nzMC(ow^z45Z(cY8q;eGD&Om%l9y8DfQsRfMN1z
z{5y!-kr68fybiB-R
zi0Fatp#tkj!d6qfMB3HR-*10kip^Ntrt`zWGW5Gp8+8pS%PlUrh;#HAf(dSoY*0>1
zewH-@Zw3Ju;+$q#u>bNGyEE%enXjU(%vpfC^xrn<>ZC}7+uQTEgZ*<2Po1A+*{b{y
z0AYok`8HBew1`b
z?SgbSNT+lt-5}E4uqe&Kpk9oRjMoBe*@<~4cVsMy*1mYs7sI;B_;Jut_Klf1{W
z^viCBX1IA^_6^0;2cR(CLCoiKZsOVu8ldr?Gy;~vVL15a#YA&+K3)(i6e}ZwAfoe2
zNrEhJn7j5$G1fJ$KWw6NLGp$SPx?`8VZg#8FrHsToCUSTN685cEnmDW?ZyB*5s@@B
zmANF3zTtT*j!BX~w8v~JY+wFJEtk@ferK-CMP%TmB^Qf&Jn?I_uWzm1%DGtwDshu_
zvXtvsxN^{ooxYwVRi#!`olwS77`L~AO0(nv)3kWm`!5I(9>o;>AnYi%)UTya0*y@K
zq@R7WdZN6NxED4bQ{PC`shUmLINUSB&+p0-E8|zmSn~BzqP{xpc+(1TBYQd(S=pxuKOKs6R}#ru
z-jP{RyU^tQ@(pMT>0y+zwg98Gi53gF_Y;LvIvT6&f5_S+{f08Ci0TaU1Y$l$Oh5k4
zX3-;RcrKK0IfRPI{i>@*O>qKI%FC72sT=}Nq4W3&t4xozj|aH{k12Z4<79Qfatf_Y
z%&!jBYjytRHr!?j7O}ClM53~a%99cqRkLj-sA!e^H6*hH8$8+w-*XAsGCQVvyW6-!
zDco(N5GJMX(Na8CPRxUVH-8IxR>saW0ON->{N4M-D*wg((lR=KDnh8s6gVy$spU$i+jlJ#hUXdZ_2+GUOv47TW`9aQk7YPh_xd
zCF!t_RvBLOP=q{v_S0&ejpS(e5jA)luhM(^ReoaDtfZt6MlBSle?T{~h#x+dyO)A!
z$iJ3<4jQLQ;1M}^
zeR#rn<*B;X`D!Cu9KCS({IGnKZidoOK1s!o!8~-c`XIjOL(^Ra9TC8QAmMl
zU!t4!_F(JgLotyR&oSyAY*Hm7of0jCdBP1a4TGXqll;Snb`Dj5GMf|(F
zo$a{uwKy3zbI2^mvd2ckF1{oNK?+CaIj^a1P|YWw)UKEwNw0I|_0>TmKvdLQK%7n=
zB8#n5v$w{==cD9TQK;o2Q<`?c-iUvfddt;52=?X%?tVB|iDWeuEC@76mH+0NsWHIA
zb20($*3UXuFLR_O_QsFAyqZnKkyzoO$GRx0?%F!_D_5vw>mpaCvFf(4Y6IBx=kkV~
zHzUbr9Gv&&)G*_xj#1pyGTy0L#^B#fZV^@Y}pT(L20+;T>^^o`bv2^dwlDcFI{i*@XB;3W;5_=QBgOV9#lyOL;=cym9I4kj}kaGpQ7O!r4w|f*EA4s~>
z>rZmN8r>nNn#&x*MX&Cz++29S1>S7B&1GgJtlzq0w?_R@^kgvg6Td}g<-Y<-_jJoe
z_fN4od77dlg2Z{|Cme8w+mGAD;P(Y=?^+l6-MF8(3EuFnDb#35Ty|sL$f5#1e5StB
zHfiWxB5+)8h2Q+SGhty_d|6%d-Da!uHMWa7q01i?PIstkLzc8b2U=I|2teflr%FO}
z`pee)E6A$a>%S#Z_R{=!xbgRXniX^d$LpoP3fx3!qh8AH^4HPZO^02qnFQ>K@9=~g
z=v-|p@@48%1$8aeUd#BZ;mx`yj+ptK&o<(poi0yXEL2x75*#ldOQCPi(F+3&N{1{A
z9OcGeoZ#!wmHA*_x)j?bAX+~}SY@H!4Hj^X6k(A1d?$hASzvmGj22r>Jw52UTbf{N
zNq2^2AJvfDggQ}mEn=$h7=c17kMM-kTN<2QKEWxecu4gKoBOmHWLJY?D^1XCMRya=
zCayK156Mdr2!W05aL)c@V;_aKiZ%1u*GjLyuz(P5lC6l;)lmRmz;9pF(cZK#N))%!p9U2s*_y#=W|VTZKXm=~cg;&8ZOLfJ8Wm>e9Xhzp=vA9MoSD<9q--?Jdd|dDA6#gQIk!F
z`A`Lsvdu=8uG%|`-Et~T(=a?%y-WM2fya(EgCSw7hj*RGhD?)scW(nGWg51116Uv3
z!n{QAOOaehAVT}YHC}q?2vzfk{M!5_0?$mi_x+k_CGbakIiQULN6;_jOQutAGK1{i
zu-PYp9cm7Ck`8OE)@Bp(HkWq@&^ElBe^=`7s?bnAqWHud=tb4=CquuPHe_S6wv#AW_jE$-2
zDr}OCjK4A*Y^LZGimhXQ^if`(`$D01I6Px+5MfM&BS5zNr1?
zVfMPg)K*TYV{0jyV;7fJ5lv#Eit&+ZuOX2`7f^)zRS2a{z1*X`IxSKA#nU0mylG~Y
zy~N%hOlXL0CK8Y~zw#HMA<&hN2<l@LB><(fOx0N=Tu3=ghSmZ3&DuJ-SJgMPW;f|!80xXPO|I){&S8bOo4
zfvZG-yqWTa(;r}x`{*T|cM?%eIFXJYxT=Ql$^DS4U7zq`dIg`=zh_Y?xaN0E)GeC`
zvo9jP2@Q?mB1sH^4Z}(?dNYo5PcOQmGj8_Z5zhnbhR@Ihc5jwPj0dNu
zS5za`U(8GeiNZn$52yVbJI%F;o(DW9WTG41)F_3Axsj(j|UJHJ_*8f{5
z=pJ*?ZmkkYL6Rs
zQBd&zKAZR!0m)-({^O5XpQ>ICcth|zq%ve(aB0({1ik$AOULwbEB^v+S=7!7$P;-l@eWfO
zE=0iTa}xUQ%T}z%pc$5!&McF0%sBBVGYs#f9_(LvSj~CKkhDq77v;Io5S3V_5eYYwd%Tjhid^>bTz~~a)dJZ6v0J?leMbP&)pSZMeDCfQ^3X^*^B*sy3}C$
z^7f`XEqb{U;SLDhU0&7^7g~@{Umn^E7J}_>uXa9Mul~D8n4C019~NaSa#x%U?%tlG
z9P2B(c0B(MpZTCPD`yPxBH6y>e6CuW!dteod(l|Y$6j-oQrSlpnR@*v1pk$f)m)==g#xnHMEr+3gS8(M=llcAqasvG*(TY0(
zmqkgY+U3Eglk{imcarc}_~bO&UQE}BQx($d>l3U0EnfX~`r+-#imU5VHPJ$Yw#V4*
zT?s*DW)N_&cks7&(-LZh`}5yue87`Yne(H`Rkvz3x<*oqZADEUCP8MV1Sz@m(F2dR
z;y%1mr42rcjLK~<`Pf3fYcH1?v->Z7(&m@(X84xk^4*E6gCXwwgO!gO)FqZ5k+p_{
zgB{wkRd~L6&7`(^q3PztQyRCath{?YD-bGiPB!
zP*GnO-O=OwCoFK>!fkj@lm^kxs;(^tWM_&|smvP$@6_f!qH>RO-k-^~<>8<|p75t7
zq6w1Ov)Z)hMsP9LiB?f1H~?VAA6il~%}k)G-3NQb@kC=@DwuXA3Z>
zaRZR$NSxes9g?Ji@!g()o^cwg@Z6d-))a+6SeW+HYcQ+W3VKA
znwly>Lwst_erp6>w?L@&jwk1)_$6K}4^{5>+FsGAlX=<5xzzDS@7SE@nyn301_2E%
zVb`@6mm^`!zlpJ5TE5CC3OF3X5VEuTE*MXp4ii}anTGdOtMB}1fkq2z5U6?h&PI+i
zojdwp>z>z|bLDmTB8TS@d%@WPa$-FG=vB{DdutHYbIWIKC!*jM=Cb*~iw{qsTd)1q
z(VPPha6yiQko?v~=7Tb`Xbvz1!VSn)1#>^*^FLH-@|5!MdhSN5Vq_%jZn$S@9g$bY
zW%m>TH=23Dbu)1m_*8vskt8PLS>nmi*G-GATk*-9KD#X1s0Nj
zCQN1NS^B+`gH6{gK})qr+3;O`13_wZwZ=}|YDF71lD|$TAf})2=bUtTblvJy*zHmW#(I}$q1m29&umY=u$~2{6?^z-|d34IQa%^4JLXmGx_H#a=o)G
zW3k+^bc@LK;`BF!b?(xMqS>dpIVD!gVJo5k+OaBccKBB}{OU!X`JdUp_n5V$n~?vr
zJpR}55D{bKWBiZzAt6D)tLyD-$;fM{ZD~coOQqsyVd=#P<6{&M<$FiKt7GG1`JW|H
zlL$m^C-M-bO~9+-XywExAR_udN}drWA*U!REG#G^%Eu?fFYxfN2ty$JqN4I*a`H05
zGJ-<#l8pcVk%#O4o94?XEGB{+B?5r~F*Q9ca)o8@`{jGhxK0{P8HFc!y5tmi7=5PVCP3yeS5quj@TT&Yy9rK-BMRtU4e!r+KLW*Vi>#
zKQImsv^rKYwx*M6rns=V6}8U*Cad-6xSpn4#0(O
z``*s^Ib+a2DwNL6j}|{+?dV
zU&Wu-Y+$07Z5fX97H1ZN86VnJyV+LJ93qC9+QpUCCtdnM55x}F8mV>yF3=4EPq;2x
z<`{~#qL7i{c3na!bg-4IV0Ow|!GowaO`-wrOy8AUJ~iqWj}z#&T$$pZsDu@(`pd~O
zWXJNUT{6Q-)l2(M^UEb^J_k$wXLC%+un^Z@tc%_(=0r6bL#9WG907tta5NR*A66qd
zR6&K5(hOZxtC1%Pm%u}K8^|=6_di;k%M{hS+%$R0P4e2^$Pd+22&Eh_@!qS~)H^Ef
zm;J+_@q;pC`}LVoAjY|}w7%pgHzylT=?$OLXp`3tc(LKAARAYP*gRdW$P
z-Z@weRp+;7^l{((o16@`9IU&uE?ib_kMqmDh-8cNWf7mS&6`#(jnii-S2T{UrL_^7
zu>F@B@ZZvqtcipft#}jF*%oEc>*YZtysyDsjRjTQWFG2!LAQ!w`pd@;ZlISlVlJAUvI(+!yCnY19
z3nfz%9iAd-GLaWL%FYbK<@u}?_
zUihV{#=h;J|FHYd;pb>?L}664SGmxoKl7(uY2tA>GyjiG-A?V$^X#xIbv!nW7h?w=
z-Iif&CVMNt>N+QgwAU*2l5&=0PnD&wZ_J17Dk?>psfX(pPPQd5>CVcS(J%?GS`Lkom9fWPA#&qU3#{1#il_?DBu|Wc;_4?be07M
z#SCc{rC`WeQwdd#(XdmBT$zomG>Hn>l)3AKMX8MTS`^Ss5lPXw66~5{COJ#wqs^jH<&X?WZSyT`Gajq>;%3nUc*4;qI@U~y7M=%xuTxEyx
zbyuYE12cYZYm#$npR3*PhN5{^M2K%u7C`@bE6sRmOjcvV#N=)xhs>9DYL2|v`Q+hwyXRqS?dk0I
z&I{kuFf#Gz&KT^8Y)O;2-pRPDrm6j!Xj8lE=k#!^Y>|sZv#tz(7fpHR87aoAjk&F+
z-zLHPSN57h8ubd$9yIxQ`c~n##^&<4^c@BAn*&Dd;ZWTvnz97M<>h211u*
z%_;VrF<3eell2IHTc^w5Oa;&>U~YX_+v9G|i{szIW>H1uN|eZC>)p74YCJLkNTr~N
zt;XlZGVM)eRS||VJ}zqa%rafN)?SYwCnUgTcI$)`sVeg~ctoX}oD7`?rxPCCeO0E1w?B>LkRoAVHsQfB_s&H_>i(*}(e6#-kR6_AI
zcRsgaI`_wB>|U3-w{zlLQjPFmo{7TJJ>%Q(1vja2Qoaau+f;NvsMU`PHi%=R+-Z%=dr;!nnQ2DQkd
z{Bx^+@TN?Eg(^$dzb6!T9aR)v##7ZEnR3C2(W~MYPo8iAk~WkB*fv4}H8s>`hrA%0OzWGQm=9(%46ToP6~!imj0@oIvb
zN)#q3Bb{iWjlbqqFczG^Fdm2AqY9I`r%Ytqj;G&$0tzrxbv>LBs?*8Oee~Mla#?Z!
zz+;=S<=b>8oSGo;8)v|Kp`l7CbWtv`;ZwTvX}DJ%H=O!SOvQ*6TF*BPJl5bv(YfPy
zP5n(7dNmZTCvFvc+Ua0_$Dimgi#+UEDvQi+B1z??FxtELqj&1HqEcFO5pv^qOYV5R
zw6!;Xkao2%!HT4^Y54GoVxCRw>pp$Ir6_JT_*(g8|jqDP@4|E{83o@
zP`7R45m|*uz1C)Q8+}7htIqM(7>B
zo`Y$fD_08UyCj28VdS?I^#r&}v<1HB!KCf$CO9B;7y0T)-hZn6L9XXt)H=Oeo)t=1
zmip^hcQvX^om@NOP3MXctxy%E(OrUnBVZu7`eo12{kK7si)}fO%o#$7q!tH?JySzN
zncwp2wfus2=^A6+uNFf0DO5_p_M`$UBQExsl)^^BW?@vJD9|4C$7mV0{Z
zx+k8w{x1W1^S=y1HCORJ8X$^air^rikkdwBNLz@FFF%rfUsq$er{q^0&
zf-)&nuG5|ftQPy;4-3W7Hj1O~FXHfhv$}ZoPKrnfo+XKlR-YrG6gR1j(!_A7l4!Mw
zerLbBw$DrVx2v{`r7{WYb_Fd~i%heoC@yJxRe$Df>;8z9Ugw>wtHr0)kDzHOW3H_D
zN@1&k{-K7iWOda}vsE=xIfnoEpCdgJ1Xf9^nF{bU5l0t8Wn2vf?vH8
zCdzo0$`m891RaTumcnwCk*d|gD)z9DmZ3#Tv3#r>_HCC}8(eBu_Z`t(MPG6gnTn2B
zynjtVs6W$p^Jp9Lqg~&iVaI$$Lh=NXvJ{5vB=|4*lqcqy)v$~sNNi$xYLY0iF;7*r
z+M^TR4jr?r*VhZH!70D{BTnKc>GQw5_AY6`u8VC`wC(W+2uFg^gQJ2tj`1J{N(Inw
z7fTFUwsk72ic?cl{A#jwVhYBaqJN7yJDM_aehCpW$|o(gA#3s^K!V4yI@PKn
z@irMpND21CTZIcb@eAm;d0TAg7qUzY*%fSE?+<-3qs&P`r|iMeFMlb^
z4~No`#_-#&XbQeq>LeZ_ma+qhJM?AV5T$gtP2PN=&GKgL?p(JVY_=nl5p|Ln{FRby
z{;9~DiW*a9QOYx;tcR#kaFm8Kh6`!wOdt5|sF4#cV+&c=nWlnhix!e`wOAU#pa+%~
zdqAiRya)4RhLDjmtX*js2IHG|^nbf2S;&BA%iD!ut%LN6*s>6t1F_V%#8d2BbUgt$
zSm#yW76r&FhzARTtd%2c6;!bHf$_F(UFBwcg8J4;HZZwrQ~XswCskVxaT@rGY5{fF
zHT40%UuNV(y~#`u{9WB1$R7ulO>u#Q3s$(Qd+^)(mdJUv-UZk?y;_!98-IW=fK+J!
zTGN231Ii;DWdujC8+Y6o)Sta!cad4s|2pxK*%iU9D#4ROCn067|0_bn0jMOV~K
zUDjK&)87(B-t~}yOWF<*!1XT2&HX}VM1g*To-6J>OEU;)>m>*)LAl4yY1q+nl|eZV
z_?dTI<8J$|K$B9z9(Ilbfm?}}FsN{wg0>I4f*RJdE~xPF2h0J|sDB`UQ24tExqRBZ
ztePloy$?(<6>LH-n;>W1IZ$}-Iu!n8a6#x(A8g8{4$&{&-CzoYroD40qS|5v72sD5
z85bVLVD}l8UKw!aM7l)}#}XRy7sK4Gj|Jp#D59M4^b+^%y&1_SmW9>f-4X_Iez(^K4YLh)bv1C>el5l4{ffyr)s|M-|G1q#iK{5d!ZSr9O{#3MT`>Gapczx7J
zxc81W>%8s4VSkT3uxMzM9P11>mK?gu-y@gjKwiPkuVu)|uI?^}E-z7oOoJ*|>>ZKJ
z2w_ZUC4y&kE}Y-F?e64!g+p*jYcduVI}A_g3c9K$XXtLo?;gpMX?gHLrd*qlix$Ql
zpM7{WkQ#V-J$Ss_a5@g%sWrSA{HQV>A@CIzFb|di^MBxY+wb>qtNo34QA?!b>u!4G
z!7o${z#8fSNo4r(NiuxDTV4ElaXR~}7`)M0wBBF*{wa<&@FQjp?zHH^v+g5l6i+oC
zETfx?U+7&wokdd3(?kJ|7;cA`_}e_hSSYR7$50;Dj~Sli-@BAhbHqk?V#P6gKS`3%
z%Cg@OTYqqA*flmVdN}J5OqM*(lK<(Ef$D{@NDFv{1?%6!V5!O^yYZbv^<%VU?`xcx
zq%iEgFGQ)+1m8r8ib)EX660a|8`pt+sXz%o-h@&USz^GH8FBEO6c*6_W0+t<2Me1`
zB;w^wR;J#cE8Du(y`0muPlz-2-JS!}DD(ou2hM%_}(?5sS71qYj
z%M3$)`V}%$D7;A|XA@&)BPlbm;3`z{ze}F=-_E2wf${c55uLR)J0o#ws2PV<%;$ZH
zZ$mMaE@E%WO!i+W^u$_$O(gWW08EAcOGrcd1+)<~|7^C7VO5Wrk9GoXBPI3MpxxyE*$L{wwdVgFuo+kBu@%O53E1!tfEj+!
zx8;fgmJR|pGM8X^0x5ruS4)rEHW0q&SLi4MwN?~GJx)bablV09P_*0jkn})Hw9QtJ
ztw`-=`|CTy8QGTO4RVpl;cz(5Z^$${L}~QnoAm1c_TkOly%bTM)J0ZA5Bo^uRZ@y9
zDpZ+dQbiAK^dbJ3igNSU!@JYIyL(YyY)X^50(u&WNT@0)N_BtU-^2djhfP++JKgDF
zBjU!in{8IsajU&{#ygrlA7RZ*hGuT;dF1tUFdoNMyf@m<3(M0+m9g$-!vaS5RdLhI
z6EXLA=-lA<$dqx}yPwP!ExM_Cu9B=G&HtTE(A{^S+{8Qy1D~4DWJwXv`v(2m1*faXy)4p3R%=a0tqaYt5xhWX4WE`ASNO
zzEbEbl#EBqJl4zF<7D<`nu2yP9r4sB{FsS2Ni$gLjB_*s|E(1a8&?VvZ7?pmN}W}{
zZE+~0t$=^_mFXsuG`qH3$RsDbf8N8buYV$nlOtEkG&c%>rAbBi$Wm*aKLEI3>UbHJo=j3
z&AZh$^3jdTRnRDkl;b@PA$?UEHzR?IwgV9!3Cr?xLqO^o5-P
z*14ItH<~3f7|=wwEd-k&8klyJJsj6sT`LVOqEPHKPAJB_f})E9D%gPUFyl%>GH~=z
zu;YI~gaa7~qY0@r)Q7j~3I6)7Q-
zvJ$lOCjygAR>z$5697x5f}=v$gS?FO;V_wlCS{QjO0-^pc#XNsZ9ZFa?u_g-ff@;jj=WZP-ODVFe7~vlcmgkN73MkoAisYDBRhXX
zq=&#@c!m&w9PdFf8SXrnJm@}E%lN&)3Jm<{Bi!(AK8pkwgtUToAd@<0P2N*%L2B#1
z3{GgftfL(4D3<>CT_^~+8G7xhOu-uR=i-t&coyj)y2xzIKqJflo*e!{vk$i*2luK4
zDg2j~1CBw}ur}Z8fw-Pe8#73`c5Z)I2=x{q
z!sdcE<9(LR-LH^5Ed)xs(Wp?d@0&st0Nl$jzfi63`JB-kJjy)q9^|&9yE{M4jVDHiWQd{B=4crb$*XLnNW
z(gO5|frg4q#Dgp0(X!2R0f9A+tyy;_q|gfr29s&V!(jgdoVb}qlVz=&2=mU3Z
zQ1BvFWz3iELHCBQ_RmMV+_b`UX+Su(UAJ87Ss?irpyJbG$CnBjP%D^|pUeR&Dt~!3
zk3^P;vRuE!#QNjy!0Q{G$Z>8#hl@6cb+
zRP8)Jc0ZTNI?9_ef2Bv80X1@Nl6W+(b_A
zsU-qe33ZCpN)L!?k|_*}%Ar$oCl%?6>xYLo{{w!`mfM$sk^&Ql)}R8n)}RAWNCG%F
zmtlDVD1VI@*>dEz@m*iRym6`|m^gU3Di6Dk*Ga09inDT(s*N9@8EOVGj}ed6Dqrtz
zbb~|AQnFtNu+eCs?`|Z~F-oEzZW12f-`{-mK^0Lc%1mX^{bMAx7I{`gS(*!_)9C&X
zeTv`je!loaL==h4fJhR_NSahgtuC7y*nf21-Kjk8hQU45)$UHi&)8ShAa-{~
z=kYrPWpQo$Bd>nmrD<&IiGN*taCX#=e6Sig>KP;2}DTyGL4hu*aiokBn7Y
z;D7VKi8gR!*R=%o?B3G+d8f)a(6o$T4+ok|XWkKx9lO!ibf9gziS7_^>=<|`uUUXm
zt8~Rf7XRQ{M|(l0)fWP~fyP6%?<+f02l{=$lU@jWIM;4uv<~5)@l*ADFG4yxg0=*m
zb#NQ7F8k~;oU%c!uF1&O*2w&stQ1D+HGh9wRtdCHk)>&LC#5J;wqLe8q7TyYOA7Im
z2nF-q81I@ag#?iW()B+?Ep$n$@(=d}D
zg>2CNZZ;nf$C8F38vm$KLwt1GjaTHF4n`Sw?LhzbXkGh<-Cdd_c$8iqWm~@Ydw;gy
zjpP2_?SS0#Z{qzEh;0m)KN3u-L+AeN4i&9>
zw}BC~)I(IDRs2*IhT>!kJ`G5j5&w>MWdkZd6oPX6&(R6WF
zT;OnPWCFt9qbg6l;u_vzY4DMare28PBlbe%)r*@+5I@E{XJ
zfxEkdx6)vc*|jXR5i(;XK7aO}Q`CJ(V2@K>6KJS7alOs9%tZkPzM^~HV0(EPrGVe;
zdeh7|mH}xJ-_xCnnL|v3>_Xtv-Nj=MA^FncH1I~SFd5e`;ip=A%!tB#GkJjzLv^&e
zWwV{U4E&gN@>=o}v~e9BWFl0`l(Ddq26R#lCz=ky_F(?D5>zTmU4N|UHLoZk?9sW%
zQ?uT)vOPG}@i;*(Xnj~mrE(+OAgr+xbyIhJigL&=nf
zTKq3Qb0!>2eI$iHOT!$gU``1bg>N*e$E<=W`~@$9sbjL_^*Dj($(v`Gt&d&r#uK05
zO#wzMW45xXh8`l%Ih$6C+ETN#=1XVQ?5ui@!;DI)x3S(L{(lD-01qBRb_WD<-Z=rA
zP^hL*ibB&sEeot)d*7C%p>HS@=jQuRbYoFXDbp861RPHkc+$E2?k(JaSNApZ>l>I%
z#~-J2=%Omx7HSD31vzI~%BLYRWuf+YAuy!q?)7X8Oo}2aulyFyH&kHY=!Jw6$O)Ea
z5Y;i&<`_bex_<{ggPLi^SiUN|t~PK1#qUQXEn^QYeIcu`1Ocg3=;r<;yguL@=cea6
z7ik6IQkt#s4@B=zp*L}D5F*QI-_>=s4-H%B?b1uR=w*o}(Ys(08MmJWwV!)d%G?a^>W`TUtbg&7km(2GRIAX;~fJYih5q~(Lx(uh!E@1h5MQDx}S@(Xv
zRIVgxtZ8;<-h>|hb+Y06s}jn$gr)r*cwO~0RlckovC~ee)0T93EmM+eI0yl=crT0~
zm>5A|?W_AcAldywYfqCKD_R_=t0%bkegOIqRZ<4Yyj6K0GHLwh3U$P~NWNkQLTh5y
zO}`A1a(~|L27-keM_$k_X5zaFz^xdWSwS2!d;Ce_lRKWMjF9o%2^{=IW&BeMb~8>b
znSP-`j;~md%8f`$F7xcN*#W83qXaa}5d2yWc=#92#`)!J`hS{mC$&&+bl$Q6d?0rW
zzrOLrN$s0ooT2z}w2Z>9IZ*MC$>P#%M_;j#DSsg~%N`^?&7K%qo8sbqs0s2lI*nQy
zdPOMx+X6qv6vQ__Vn7ksF1ba>(?kfPyW^tO(W_LdC{Y6FZ2QseC*bGyRX<1*AqyQ$
zCsdS8W!qu#QbRq{<*}%kjU(q)9zY~ff0u62uI2GCJq%;GuH}jvWChZAMO-@h51%Y*&Wj^jRmDH94O!E10}FWo9~XYdH)~D2vw@*hLaTC_a1*_%GZO5zhkfrJE}dy&;*)ogihk%74lpk+3Q&6n42XYW
zi`U1D)M0+Z-;D=9yh6*n`s(OU8hm%UC4Y8VKqZ*ZIje*0(u5Mj_{Wh|6Yy^g2wai`
z`_b{S=jO#LCzLdgL9la~W2ghAXs}TlgO*H|hf?=%TLmUWoLmKr%VWj-k_^{1M(8Ai
zNdaw6Mws#n>yv+!`heVY{l-;vA~GMO{cbh~=xDFXauYEAh!LuYB@N$hP*hOf{C|GV
zxTjd&`Km3rOH*L81U5^^9BJqdWPY`2bqLOMIO9*fU1*J6?((XVhgTKWTxi}FMQg}K
zX8C@^k1q7k+q(Z#_`CRUH?ac1))wBjw!8mqNlc4W&k}3w*t;)RYF4?1
zK1YH7m!c>Nwgt69f`;i=JHJ0o3YYNl14jWjmtlDVD1W_JO>Z1G486~<;4jdKBB>7`
z$RSB>mvF0eOi0%+1*)NnK%Y)OV+%hNWLdZBPCCfgo*JJ
z0)C03Qt
zmDHmxk$<$#Cxx`mnWa-978qAWn>=YGR#r0L)FQzFHt$IiT7V3=Wbg^N3@6d$T!gF;
z7fGGLUdaq+Db!FRJP8HvA~?aM(a^%#7~!EH2?$M-Qp|{#5nYWvA|S*cee7awF;5DJ
z9nk|C3euEdr@(kZh#vDKMhmACvJz6F43`DQ;D3rJvYU}6#M5OVsqth*i+Hl3&6#Tc
zBJ!CNS{TQWBrXI~lHgB)otjUCs@`WRHKoQ~0ZATz)~Q8a3sT9L1ng))UQ10ioMX(X
z!iEl`Q`6-GvhD%J6IJB5(iA;+>|rdHiiNc#6$^6-KC_3t$}&hr2iah#LJL=#VXvly
z8Gm$CAGz}AD3zQ(38QMdqx&gsPt*b(+
zXu#Nu7u26nf4n$dQ2&NLMl7B8)E|9beSe~tFY&nRPrqFb)W19Xd+7Sti;LB8vAROg
z71;IfhpWZ)@_e|$vRV)PI{fwd?A78ceXNLxw51Y0b$EY?7a`9z?AG-CgdcwYqb?Pf
zAl2l}fuxJ;FJC?#LdqH$V{%v!El|kARZ7NCLOfp<%ahBq^MT~%op)!e)v&yvaDVgf
zFJD*3C#$p7K)fF4`Xi)(YTAz0nZ322sO)rnnIcMDb}Iw%A2s9U1>ARIk~KjSXGz#XDM^6uCYp`*7S|iP>1bKX&-+3^Pl1T#?;$?QS~3z
zATW^K>qoRGWmk)m=2Z;I%|hCSK3p>mi7cPlfW2u35$S
zWPB&@ner=pn^p1iMzEQa(0`Z=6DU!w4IFaX`*eLmDUP@2ds)tv+|5uG++H$_kI5yv6z@g8g@@1gxB
zc?bJQsR?oVhS8@+%i`l6@y~h^-@nuY?6DJd+qE8x?olXVCMJa*YZompFtn@ARZ
z?_c4!#Zp|MuE0cjNM@2zR$8rgwxx-b%mZz(8^s1M;Bv>>XmMFsOk18GS%-d=sg^Xw0-tvTO
zsv~|lAL&co_&p16_`%eJ&{_+v*dSf?r(Z=!RYyB6AXLJ*CFS1u7|y7Bk5O`^Q5?(##~p$72Q(-U$4jONn-Uw3V2p{uUsKo$!S_FXX)sS3
zefmXHV@IEVvS@qAyAI}5Ut?{yfEXU5aJ9OipyGr&a6siWW;uJ{Lytl~avkQW)Aa<~O=1bjtW{{1jRO0|
z3B)D1%yM%4#xzG+6MbA~AQ@7fjyisBj;FlEB93nlUK)P0y{*qr^tPAmQKv3eXLIfd~h%6g234zTXq9!HYKtJa18OAMpN_
z1jSa+4%?ue4%#j3;+5wr-N7}NEjPTnHD4UsT%#~lLe0Gn~(9$cO!-Ii;kYGp-o}-5Er`hv83^kuzfgE^AgkehK@M|`OTk+VJOAI3`~gTkAfF>#2XH1NhV6PBgGnS
z29Z(`fWsTF0EHlLA88iK?*Rnbo|nM)#`9RVh!2A2&qF3QaH5q{Ngo&KVhYu3%si_F
z=IP*YWyZ`?ffTWfc3`*}UDrh#WK^>`Z!rT+6kprRIynAk(|Ir0
zO*+icWIE@Xb0p+%JXh$YP2iYqGB=&0u5FT7sM5)NlZ|vq^bOC%CFZkKm(FylL<(E
zuZ1DNXbOaDJmvEXdnEAU^n8~WkOmkRB$u7cLeFJMI3a*}z%E;+xbW7OhXa2&cZ+;h
z#c}CaTBgijz%^|qW{O3E1Xh9Ue8~B{
zAQFs!GI)(F!C={h=T7!hB9%kTy)Tbgd?QZPt}=ZEUGi)Sq+^XQ
z!1=_$gG6r@@(}PMg+dKB)*LIR+p+21TmgKM(Y
z2LIl+Y$~ZIV*q&m3#%#7@|S^<0uur@F_(b@1QY@@FqdI@0w{moT3e6XHWq%LU*Sg?
zh*{o5N-6S?bdw^*7TZN;u?wVq(8!t@A-3g6@=Rxc{hq@^%94g-PqN)C&=-rMc{z7}
z=TLlcv*3##FSvicxxD!9ofHcpSe~YGae2KEN-=A)g-I=wDqUQz7ax;1MOT;0m9lB_
zp|AEGHJfXCzFvPSk*te14ZMn^>H6hwmp^^?PS{1xaw82c%UPbmmV6=H4$x^-#xf0m
zZefFe*8S~rB~zXBw-q*2Qe43^PnsqC+%096R5V~+{mPZz)qUOUmLln>_kJ{@X#Enq
z27B+Rx2=lx1NHX}b#+xwjl(`{4l9OLHaATRWNqp7x@mu@@frytZjrpMXUOhCFfK<7
zYOw&$S(Xc8@DkZlxlFFSD4|)etHQTBdLm)?bcCr~Il{1m*sE`Dm(nJ^AANrdE5ZA%
zf2}tgzXy!Uo$QOp?R%JNBG677j1k<@D-fBNejh7wqr&NE=z)Gh}ze1I!i8w`43QB_U(|Nmy
zGfs9hDa*`^kryO^d-Tc^8|JW~m=Su8JIw_XBF-H@v6&VIOmW8D_vbc~uvsprlTOJ{
zgIc9hU$~3n%os79Bv0+4Qf9fF@3IocCNJ?}=wZXdD^ZEjVloA78(7a7)vuiMb7We<
zQY(L=!g$g=v^C6#8L4a0&g0gts{pdeDE?Kd+Nd)`pwvkWCaZz%FPhN8$&(&=vFwZNai7)FEMN2GtB
zGGWA#|CC1;%B&7!9wPK;mVrq|=$QmCG+aHV$3*DigLq=nA)WdWKSriomVsm*v7_~j
zok8XQim(ITplp;K`IOg#<){cbnx)y8W1z^{&&Auzi_aG!ph{zyhR~jB!7NNHw-+CO
z1ss!H4QT;}MWcoU2HOF^LUYqNEZ^)2v_emV
z7?l#DmU;-rVx#Y??cU*6_eReWpty^~*AK7=LOUyyU9qh;55v}!Eg0#yo+p1afpoAo
zt}`ON0K#<&xeJ7Tk+cxu@BBL0yKjl`vg$hQ5iWsx8WKqZTV!^+#Ws#;0$DPX5YlF3
z!ReC$FwRKFM7{iX_OSx8VZ3!}$=Q`CONbye0K0)`mO
z)F=|=MbWoaHFG)5B*e}MhQYvJ46$|wW!}|&fG#Mr1Jx7|vli4pW*2{Y2X`7c9>LoI
z{`4(yLp}>{hi+&NDe@@i%rNJCY&a6Ix8Os#{i4t|F{tCouj9y1X0Q;)GY$ZvR!6B_
zJ(e1WpdD0_!KndeqqSskXpu^OXtw(U!9hzXKjOr}gEb_d0PiOYAli!-kn$l`04uXy
zeF?%kU${B6e*fmsQ-gnMC7G+0z-tv$O0su-MVN#JjaD$KTv!ZfSR}%zJ1>$1DL@*i
zgqYByrW@<9iHcNF?%QT}18ZmcF)|1B0CFJp=%aedr;Pe9R3!6zc|-nufS8R14zPHj
z=pkZqkmU(8LNP)`jLx{kY;1JVKn>DRvBYH4fGY@_I}Sty7%G1>dwnLeb*(T4OeRQT
z9(-}FYz`u)m<*5o<3eW+eZVY6J$97!Q=0ft^|R2htM20wfQCh@PLe~C^L3HnQL?C~
zIyncc&mBS
z6C(YLgh)R#A<}=R6QcaN36Y*AM5hG&^0bGhxkF4?Kai(N7^8D4=A6g)j8_6t=&Aq<
z9y6UkH2s_fQYpqJD4s5c_=Dox7R~-|wU86uub<`p_Bq}!|6Jbxhm=FixAy*Y;{8u2
zOyR}~@>&3gIf1cvcsb+}kvt7oUK$hEmJ%F#KXcn5HI{!~^6ARkh$11aG@}Db>N8oa
z(F~k*vaCGKq_iLsJ|kt|i_-bYU17kT(AYK(#jcOg6GgFG*LU^$P;6XH0LSIl*FjQI1^Hx!J8xAHE396YPzsEed(M`#YKM!K;mW1yLK=!!-p6}nrABIa
zobIlvbKgSmI3LPX;;IB`@;czy+du*hc4Nz`T6cdhjz{iOMg=HahZ0$H)m1H+jvY)#
z@od5_&&D)!Z``IlYRCnQde_sOtQiuHnO
zS5g%s%8W}Dyq4Y;1J%NMupwp{r>nY9KRVROkd!kESZA`u_Ou~l@@(Lgn9Omi5;k1Q
zLF<3IX-lVV?Df=?4e+@`)9k_dD(Y-@VJevvcFi>w`=mf|u;Ea0a#w(qs~GcT{nM=cz%j%CLBbLL&`D+D$r=HU5$vpIBuFB^^t{DYRh
z!;f5i*AI^`wxIWymOg-SB~NDzuq&JynMZ$Ok3W-f)80~Htb5+MCY5Cz@3~NH|8Qj`
z;#WrO126O}fThW#4`uzS?pJ@UigtJAm_aw3ZRv_-#w4{blR>j8I#)>cvV@*uzu#{{
zpOYhYu8VzN-wkDM$raGG#oMMAy0r#SW~(KF0`tY>R(&
z4dx8G@S3Ae;1QPUfRw;2HbT1Em#7
zapS6>u1Sxs-qsj-+)H0k{Mgl>50&p~==mD|02xyuo=%kLhA*(4I!)rdvf1Lyt9n;;
zu`ih%QwB`HdS~sLRS7@=JyRR*;YNS)3E#xLI@(b$VlN!qRyQqpM??J_y;@Ry!MB12
zgBLa^7VO$q5LcXmtm)6Yx|j~G+PWfJbQ@hqO+};tyF-6#FyM*|
z&UFr-Fk!m|{du2{>PZPEx##nWpxY-jZXV(pFwk|r?pfoN0d{ran?6o)7P>ZI4rp}P
z=d}hBLq;;;Z#vw#=HLgFKA?%@`mifQRP^RibVnkh`VsWWlcReB0mYvN-RA=w2tLM#
zb!XYUQsIMv9P%umVG2DN^!5kx;y>2WyCj!^k^&O~H#xUK4+N(Q0X3I_#0M0&{TT#@
z9|1R)fy4(Bx0pEu3TFZ_IhTRN2NSoqpalJI0Wp_>#0M0&$q)rf3IQ{hfy4(B6EZV1
zH3~0GWo~D5Xfhx%Ff%hbmmxw06az6aH?rzqHoYp#iV<-OLk*7F-WDoq_WDQl=D
zNFEAtXX9e$6b7iOsR7;Xl-z+}dn+zZHf@lt2N>uI;9}?IR
zCD09~20?7?As{B0iwxA+$JO4}&iye44;$NKq(^UQc7QU_$`R`2=4cN9LaYJG>}u=)
zb*L9iXb)h7LI9Q^J0REw0JQ<=f(!t9I&y#7IsgT24LwaACU#h09S>(`sO$f5k7p8yA12cSdSCi9~(lykS`+%*c4zQ!e!72l
z_HF<#sH-CY_U#G+gMK&017Zzpr@I~Kp8$`i15mZM0zuqBk52N?f0RzJR>GWMa`%4|
zgEhkaF(&vQbATHN^dD*LfNsBiRW&tL0Zu@Bh&u=Zgjm50-GS~NZUBqlGT0kr&F~LF
z5J1Mm)%7ul+P_q;|4#ES>e5hHWhQ@MUq7JNf1el-;^Fr4uipG;*;Y`9o4uR6+dmOO
z02_NS=#jnKbt7*9%8RT#i8#0%hp
zy&vaP4q`0>b#j8Sc0+$`pR7HsPwr4xACCWHz#Sn_FNp8|rM0n#Slc|Fg0+8#Glw3;
z-o*o?B>O);FcJD6nJvg2zzG1k06^YWb{vnXe^1V%nCnptYl)w)Gt?Pi0|dK){OoN&
zurG99H=rj7;O^=H^7H*$@t+8tO8{VPZ{-dfdf4Ki|BkK%v4H}F{t?5N{7dye9?^Q#0qL{53vPs^YH_KuC72IbWYfWaP#p2e7RtQZw>PPJwgBub_moR<^pi`aQ6e)
zKwZ%vkC~qjz#;W0`Umj?IHdm|0RV^0zmWhZfJ6Qd;sS6e{6Rbb4y8Yc7r>$X2f;F^
z{6Vk`s(%nHgW4Y?2;hHE|2Gok25@NqL9m#*e-O;l;NM6H76tfk#LWp)0>RF9e?&qs
ziRB+81mLjxH{yi_wt{W=A8Rg7Sf$o~L0GAvzaWg4&0i2!y6s;O7S8T307yW$zX&Vd
z{x1lt-r+9@BkuSYgwX{5={}4F_;F7D=wZa2{y;960pw5F+>g-XFExyRHuNtDqwW01
z8a9_4&Oldpdmz}_-sVp|zvUoTd+1*c;DXiU`WJ-N^oJEMtOhqQ(Cx3HxnMEe|AH`!
zzj?u`@cIKEX?y<#VXS?A!~fg~&BsT^?;DWw&yxJlv-CICad(9}g7odJVgDrlGJ!3Q
ztG%}|C+zy-g2`d8fBiCl{f`86e{SBtyrrd~-o9)++yFLiKF-I(6@-m5r{8~gTK)4-
z`+af3cKKiU@reQefxJOh=(AH$E0Iu#)cWuef4PFuk4SVv?0aRz;s(kIh_e}^tn?JOxwyi`G)U#$1|G~nkfK!gF7-ptsPgf+`|ueCn?k>ADb8Qki3c}xEcsu
z!T8=YxKG=j&5X9eyFXkJzKH>LNzF2Ux5y7C9#lql?(85~gO4eaxQ%99%jDcbh_N$y
z^Ux)V%~I*YmZ7qLg#i)lKPyyPWE)ZvMJIrRKYCupzS6E4bJ}@;nr1^5s2?qpa>M0w
z3h#*rNAsK>XTWMxePOr{b*&A(URNvmlp~Tn{a(=Phe*G?{@h9%!1LwyoSy;n4?3u_
z_Sb-H>S%HDuM3HHOL0EWt2eGmmz!#&1?_y8S^6=&XRoY(k!X7jM(=N2gfp;g_%Nqq
z-t@~Ai^ZVyJ($}3oSwC6coD#W)95+=K;e0O`*a3DevRO}zAex9jrO#KNfpzD*$@mjG!oSK#K#zS^xb!$
z5leSttp!F>;ziSD(yMhgI+^Z-m1rlT&nxZ{co^yj4}8rn4Mv)x8Kam1WZz}CdCZH(
zA_LIH!4sSxL90cXLeM8B{-F`(MrHzv9{1)fN-ti2LSn+6YRxojIGjc{KU~HlRNI^e
zVE&w#06blRAUcPx?2GeT*2BGC)vLI#O;J$%dddGq;InkS%4((0l1G`ktKRVE6Ktm>
zxbe)s*;bC@<=8j`9G4bS#yAeY>
z$gClM!_DOoH?2O4oj<$}rR3yO8jKwL!x@Kw_}tW(6Tmf>@+Yp;d&nyd>MtFtY8Pd_
zhnqds+G#4w4~ry(rA@;jDu#`Qmv6ZcD0o-e&wFEEq^_9MO|vnZ2PP6llI~g1h^-}2
z$Bl{gmD+?A@z6-MB9sEGKf>LtINmYPiz3Q@6ENI-WQ(jHQ}8DGMfy5$?-%Cxatfv&
z1q$}1{fSFal~LFXivC5xljeR+QOoB;{_Z*tNfj1qjnpB~kMNV2Et#p^naiA5>fe)T
zLn4-Hugg8(AcLEYo66=*jUPzmvNAxRF!5B-c+i07!
zWqw#;X`}7=w-b7PSk(5s2P2EUqg4%LJ2zi5k0h1(=ao^iu9)w~4b!eI$5s_s_ZQVG
zD9p0)$`uta$Jc`-cFM{zdlOp8)B2dXshi}xKD|0OAf>cP28o>d;v`hKnEII=fcXNH%eVisI?
z{~pWXMeQiT9(fKMtH_wqlynewL(8Tj{}%rKf)9C8s!$q2$kthY4^Rcw=;!BuB%8_S
z$Xz=aD(CG#_sO2j7Ihk9MP%>|4z`@K0^|bWKDv9gr!v!i!eP$65T;2#AJ9Y)iSp`T
zo(IXiJiT|7C!hwOO3gjLMzQ5KcuGZF+x;@{X
z?-M;@7~}HQ?MXgD@b|TAa7ecIhyxa5)aqx=cf}(LR^fyQk@_W{>o3?PF4wDHx;F`qr88iOP!AyuU}6
z;P>cNtPo3gW)CQzRSi4~d>ZK^=oWXGcTfUAdrh6(B+sYh)lfgZqPzxR4^o9wPhyV=|JZ3ur<9I79PSCm22;LhAlW`19Xo&*q+
ztR%4)lOW!kmy7W|7V~MMwpL^(dCdxwL6*D57|(~j3SB{&3M(jV657aTjrZ((3GD9vU
zOS(!X2zXtUAbsJfjRQLxT&7o0;+LM;S?3qkTdmJuExHd#R&UapX5!vergUtkrVZ+b
zMaD_E^G}>J`Tyd{G$=RzN%)DFLGfD*y>VT~Zh@Y{ndj<{4(GgoUY1iY62@$N1uCjV
zqjNMY>Drlio^xs`Q{TG{CbI3dUmy<=8+3&`^#vVaePiy8lA
zl)IYT_9+*%1fzr0sa(q!$>5ipqpzUGDVjbVwJ04=xLUyPgTHB-TA|aLL>W_Oe2+v%
zlUE)fnlLXc&OPscf)13s*MbnDWVS?*as47W@#X626ZxZPZZ=_?
z13?7evnSd?EUDoKc=e8k>sZ$%$3;j!#gGOt
zGLgvk=6Lu_Vs>!Kz3uR1b7T~NPeD^%4R|iz+~30-R&_0Z;s=;&$~=CTWJKE|kjetFNPQ7#vl(YTmhD&5@sF)TSC0+kT4M@VUOIL2!AiSKBK2m2V#xaD)<
zGmD)uOeoe5E#ZE|pSI6#mi$|YYEQc4zRNXLl=%!(TPS<(+~a)}UT^el@O;84)9V6rB^s|SKOnOt^
zNP}_-Tfd1coi`(aBmey~pLyJ2wP!C`*Ut_Y?G6vLh>CH|G3x_A!(9nq7jo~IwGCZv
z;{I&14Ag8@x2Q69C;>u-~_(^VBV~QM%u0g<
zNtX40x;|qgCD!j}(j5wCQ?D^Ludc}3AZtJQ@Dnr;m_HFkPi~X-U}7%K%FN^cY@`Q`
z|BAk+Q5KF>SCL2dR_)J6LFSydx4s==i3|nv7z@X;R&|d|Gwd!ttmX|xrTkJQ=l0pi
zDB2-(FqhOLCXvQpGI~)$j-D@NcV*gz&6;d~E04yHs#0@h0zS{GO+!+;MnDniF|63%
zLHZ(EKD=i+)aOCsX0>1uBsjVgUbmtVVm=rizEe?0VI1zDFM?DM)*x(2Y}bQ5I8~FI
zJA||GcA&cO8v{P-l5+%BG9UTQXwY@{ZtM7hf=KwUGC}l#feHrf@!(`o)QjxS0o2rg
zpKXO-v&-8I!UnRLIWS)li<$MKm(nSfmcB4v4L8%S`*HWmgLi{N*a)>Zca!^xTSjy@
zkry)AjiBbba27s(*1=Kl6OjlMW;-+TD0dv>>ZJJxQn@fhR;Tca_nD1sT^Ae
zoA{+av9W;RvR7oq-iHaG
z0)bWv*H1s$*9S-IhS_D@zd3S8Eqg)F!D;!0Wh;q`Z4n>qn
zuxfWZKuJbl@9!gc7|RgM33%l=;fOarxK)3y`;kZI={xwN5eM3``-|KJ7^7`DX}9qj
z@S!#@(}nb%GC_0;9Fb_yu!>QCawOI=#_%Pm;MO6dq?5`aG>td4JGlC$ZDxU$%CtuH
zU0m^`7ByVi>AtJRw
z4M%pkyTlJ%AFQQWoW^+6u(}I)fvh!W+UA<<>J3QPvFVatNby=N85-1oC;fD_=!I6C
zJ)93Y&NU&J#q8Qd-;ck=d=20lA@68bbuCdX??tD{<&f|BtazL1XzzX`8QD}!85@~9
z7mWIbDgjp*-k=$&HK|lpIZ8y$AVYdkSEDdQI#plz8HBQ|m=*U0vsPb(QlHya+Yw9F
zjG&I2LlpGsL7=`c;mDYO*+sJxFQKK*jH)-f%>niz8EX5E9*J_0EIdYdC;EoKN6nNE
zUhHjYF;^pnm43Fzayg`iaiVU_VW3=p!YoGaN(oufE_oVmmew~m>8V}{d~obj`5;m~
z`6*`B@DnLD6cd8b#zT&%A}<`wiBVpnPnvjaNV7|fo$hyZTb5~m2O04d{!Bs(j`2r4
zOI0!>&Ke|bo!5MMx6w^srS=#{Na`47L*K&FJiS?#&rQ?jaKoJnBPPIn9l4%q^x`<4
z7(fhHU`ps%-iL3n1%`iqo0{N-H0<)c^MQk>+-vi53(d2}!-HZ5JcXSS)ByR|{Z1hp
z*HdgW+yu7kyl3lw{7LKeTJ+h`Jz*3UK>o9^>vo4z06biH)GG_U09pF6Mm3SxapM&1
zLZ_lH5YwnBro#TJcind(UIR20%LNl`7t={s2~=5{kPw|J9x-D042RSh9+CF%e(4!$
zu|$XnQM|ebxz0c-H2f!+;XP4%xBzK?g4Knyu!ZoC?c3-=b15no
zA3g$+MmFQuu61723y)wbiV|wprYC6gU|cU|UJa{v>(u-h9aPd^@P;6&B#5zvl-T>z
zYG!RIrUHr8`lLh#G4bz0s0gyA`mZ3v!^6h@n%~}LT)4H9_oE-;U@jGw
z>H~dz`GC}auj^B+T8ZBBv^OOf=BGrnMjEWgPSTDI@uAm|lrA<7>OAn5kP3ZxPBpE@
zZJ6(>=7~uD^w!FBw%B+yJ5s9iWsU(C+)o!VAO?CL#pNr~;TP0`RGP(GKQsC(Q2BpI
zT7Df1x;Pq1oA*KP*ElREC=yZd-d}QQm=#KU|7_KN$BBBPe!xCDC@NUl!a~p)At!;H
z%9$@TB8{-}=k-0?UHXyDaF~pidQ721roGf{9YZ9?vByE4*xk)j&6Jeo`a;>y2(B@rV4L&V+Pu?$cF9IFiN2Bdn~4Mha|=bZG>CaSXe*}5
zEtz<(V)_-9?W=R)(gw7SSGuGtc*!EEW0{fhocs_~nbw_r?QaZiD&KUQ291H}2t!v%
zm2qkD4(4fPEa8dh=T3dRtfud7W?5O^u5c_`%8_+{q1Q$JiQ%{eHq0?;0`Fw+x%8=mR#I5xijRsZF@tn*joYt^>`0AT~p-OeJ6
zkpp*K#S7Ut@V*UGyGiKEWHGr(mZI>+_UHyE;q9@On$wtAb4zu@RLNQDwq+qHh(|%R
zoyO9YH3BTg+jM?tR@ColOT$TT28chOvp}}0v1sgiCTq0qWcBAA+NZt)19BIC8Ya>&
zs?tTYr7u4*F)iqR-S|+*l1m$~$SdJcfKO;#pt
zQO}5m^vln>n@ZUOD`kt{;!0M3cXo63)&rnzIb#Oz%~I86m5H&Q47N{uz5*RHsAr3Y
znzm)nML@`k-{RkEHFOWX#d9EJs5}|6H5!6gMz!C%Qw6$JR$f%FTeOFhJlUyY$g+wI
zy4pO+`Km_+|G|m|(B2l=DB?SSay#WW$xxaUA10|u+syX)ZZPkwbLr`S)se0&s${E^
zD5>5CG-M5V|Hiy3oJ`MV>&0xlpnbO+?r?}P@L`)OSPf^W%h{f&9n5s#%!q^Y$ylp+
zL6c>Xz7X3Q;}nPqKT9i_=R>wG5g3yWDpzPl7Fzw%HPOQv=aOoGB~S6?)!?Q-Gq|{w
zznLpYF$6B_`05=F!n67+7q3CHIvmX5dB}8}oU4j(KqqlUjVQ5yFIEKcc#y1M^|vvX
z-pQmYd(Bw|Nks#zy)(bqRh*ig>*-$Zm!Ybp+-R{%Sjv56gt90q^Ew<2$5`?6PV#U!
z%8Vc4?5j!86)!AUTK2kpZN$9NRk(d=!HP=8F$Z}+n=0EIaNDbbLg_?y_=HAT(f;x_DBe18I=Kl@ece&76HeyYL$|q$`bSyY8^wFVPZvsxpFhS8ehs^!3h_52Y^(q
z2q#TgkFS7NOP@{VB2fz(Ofs6TXXJL_?*(%aiC9_`Gj37J2Y*6Taa}qQ4m6igJ7r7S
ztQhyJgY)ALL}_><`dtMH<2w6i_>Ca3j(BhFeq?v1r?d)xT^7;lN0ML43cH%(f;9&Y
z{EgtBISAT*qr1ait7i#0$-tV4o7v?RxI&~tM!{oWMBXQx8+GJy=>pRESQJxoV>FO-
zOn9BG25*)LrRkTM?)_YMoSN&TZl4q#-s>2B@@~$*0hrCfI0l-^c>OK?8L+Ir+OhtDozCx4o-!rgviiN4QxV)txYQryf&xOf37!O>>VTDTZD3;V}8M%(hVy^IX7d
zepaQ}9pf|3SwZ0Fy}}}L!>@g=njOnd35!zyJ%if=&J&;ewPH?c|4;8K&#M!G`9s$a
zl(hDf^G(yHVfInY-MxYISlz?{X=q+)+N!@0Hk0le7N>fb39&aFM_zVvg|o7kL&TX<)S{p?8;tC
zU21OVqY<8XP|J@btL!C1DtPqVf
zrNO;_(II^9?LK(i?=}@f6A!88S5DOxIa{WNLk(8kZkI6Mw2k%ZNH`pS8l
z>*MQX$1YonR218@xcb9agasMA0U{lO7UDyHGxQl;5%8|Q%~O|T@8F|jH_trVPB}+Z
zI^Nhzs7S2U)GsS=Pi9$pPa4}9*+d}421fP;^$H`W9Y!>WfV^5&BU(^wB~GKr5M*Cj
z^^u&{kxhkdH)&%@R;LAKnp5k}&`no!mB>)wH0?M=t?THQe6P
zR@6?n!bm?IkXh7<+OdO~wWJKbvn*UnJB5B11UPaA+HSQ9?
ztCCg*CP+nc?xN)C;>k*VslKQ7ESwIV&RNcV*vQr&RB0(gD7+LEbpFW|Iir3Y+%YA*
zid(vq!Bv5C2)~Bv$4{&390bhMS#U^G6hdrK@wo3Fn2wfNZ*5q<>MXHpnHoMqIJP6-
z8jmC_f112dZpHY%ole6y(uX~NwuMO-IV3`I5@N+*L
zl~0~w7p{OT-g#i{*AcA8uaC_oyc(%UVo+2n@ViPlO$0s(8WOYxfK4};+Q%7-5;Xx&
z-x^JKp(AeW&QZ&{t
zxMROE-{H4sQfw}3FVrV#1v#Ysl30A|$G%KIah9;*!Tu%|*_=;B34AVs?hbuQ?}Vn6
zJ=190!pz(a7O|6j6|Oh(Ug8eoEipG|tHPhFHn8
zNoX=qg8CB7)=I^ZzJbfo0h#BF*2q6WAk(
zg+CB#wV+6Z_T8PWIwlf=z)^GhLiUu)R7Q?18wGy*sggaoXncYW_)YPQyKYyUL$^ka
zdXwN;63SdF(c`Gsi7#c)*i26wi{T8jhI{18=OYS}2W=34-dJO~p=nA=I+o}Y`h8tV
z(?mq$5)H7`*jLc{N&&A)m~#o{rN%&-rJZPZl6;3$vKU+$V_vV!>4Hxm0Pk-oWsxi9
z(cKhF5{G+V50w7?Fc>MaPHVGc{2ed9481i7t4pGM0pibs)}!&3V`0s}dy*&VbMIv9
zsHYa96V~~EX=3oZfZ=P3aJL@ofExN==C*<-8)-+M?;{xbM71~+wzu_p2}0>kHCGN<
zg4YAB@jvl4WZsRwq1I(ETC!-F9IbnCU8gvvz4ecLK?E`E-^8Fj}cIbQZt
z>P||3!+y}uM$@f}B4U#hbHbWpnmc4Z);wY~)H~{bSv|zMTB{AUj!%ou67C-eXdLOB
zb9V&jrFg0<
z(ITplnb_OgsVFFk0N2DMki*Y!RG2P2y9l57kbJ1F0oQ(Efw^Ee1f~7C>wBt8p^0CF
znt`8xFNU%BW2WA+%J0`(t012x(u=K{${T%N@_BNi32&3+PJlLqa71Zmk{hHI;ZbC?
zymvf)O+m2%oZp7XHM5(cTWqM)Ho@uB?S{*bI7F;{3NS_SR3xwBR0YzsZjZi1KqJ=#
zm?9=M%AEG@VdWqn-gFr>PpujC(tk^Lo7*dYGnGSpQJ>Q<#<=k2g-uZ7*857sY0Wwj
zu*#A4?U3{b-sAA^x{1fdzq+d|l9S-fJ1r5cB7{7Rc)Kb_Ri>Zshe`?V$y2eU2Zx6^
zoF5)4jbJ&c$nhO1`KyfNXGtG2d*1cpM$b>We8-P#RzgvF*|Eiv+r1lffmsWJrt#W;
zgy86Sc6^a(zFkRcR2+VO^D7w4!@qCqVc#RaL$g|kp4iO%dKED}v{Lio8%7(W{R6d%
zh2*kV9IDpF1Ir~)nQE>&s7tjkqxL$uHw6Ez$L8~ic+^O^`$Zhi&QbGfK6I1GhWcx@
z30f7wFU7Q4)oTl9ZxQWeYvP2HRQL*iPXVtFIv|^8AJkcLKyF6x2a+0TGi`#$thu)3
z=Wgb+ns687^KiH2iOy&~tE1%WCry9S4cxs4iT_;d7&jW17$_8RYRJpFfESfR>8OrF5nQE#L8fUn&Ym
zVr@wIMGv4X20C{FsVC95J@sP^>IHM%^6uE-2k}xh@04G_H6HvRW%e_tnJ#!=uvq-i
zVC8hqefmp}xQ(5h3eKQe(jyc!mD+)Y{azK`OgYlwZqPN_s^Qao&vH-wzFYV1igjx^n3j^Xbo!=5_%K_jRkW*c?3{>{3Cd$TIuv2N>aP|N
z-#^b{#g099o<$&EX}!z8d$Uum9`!4H(kYPd
z&axBD&{n6y9*G@F&xRUI2Ev>jHk=B5TESOD(lezin-OGRCaNBP2)A;rJuSSd7vQ&#
zc%dJ_Q574|T^Az6_o9uJVX5+6O%Y-psxt}UJ9x`9W@!*JXTcA~y;1`_`K!*r7o!ki
z4k=?-?P?t5H@eMF)>eivhA1Z3=rjik>~35Z^N!2!=ecuipzRZlo%
z$DX@3!_To7>A$pp!f%-m@O{ZC#?qu#uDFKOKcrV{uex|{rn;ffPZEaYvf$}AMf|dM
zkxwBEPFfGWsJs`fy)4y&zVSs9057c3DAAr*L6m(IZSwj-Y-#Wc*9++k%eE1GOBfUM
zo>e$Jy3sXCB>WmVjrS$ADW{=x?YfI<0HL8&Yjx;x>AJoVqj{RQ}U{yAl6s+c=f4
zGve(0cY4{Pga%1F>(Vy7mk%$9mI9JHZuPs@qSLz85g-rFzu>QMv^18JFFxdH6(e;N
zU@;xOtYnx>MlO0@)J2Z+s~zL)Y4=k!$O?Ii{#Kqi($g|+gc4bVNiVN!T&X4SlVM&9
z3Q9t6aWd|IpQnKCS$go`C9#Exz?F)Z#AbPjDOXvb?f9V!ZQY0>cR;hX=*F_7kXgXX
zS@NQ)RCY(bO8yJ&?!4w|Z}|@X=N%TCVMkc}Z%#*|0tB3!@xq*l7D0{`eDjy$$(d9>W?Rs#)wh0eNjPE1natba6M;jk0MP;(YS-
z&TVdcSqBzG9WrT-4-TT(*(K{0eA2!^3ha~d&ZIOAFD7Bi{3(&4hX`d47me=`uR>Ij
z9n^n+>g#%ZHj<=fc~3TB=+I?#-RzfIie{U=q
zKA;z!a;hK-*bDiRxcBW`u`5Uzi(V=;XFMlcaDNClrn^Y$?ip_l4wfa-d&913T?^jcz(%dLFk#Tza`Q9&(qNqrUb2-T5IFJBn>&_Hu=XC8y_2_I&
zE*!_`E@`R_`qAJliNeRxuQDz7fJ=>0V
zmrA$%7(>7G5#E~McIazY=xWwS>*NS6;#(qRHhT&W>I4HPGo`hyjw>NsL%~pg6~j(n
zFfU$zZEa
zv#*kotjgE}KPoPls0U33%zUF{u|~U|jX3$lp~NlA2*uF#+m-3;X(9=67CdSa9r_aCS5XS+-h`_r
z_J=2y!7wmMIWG<)k4s&(0#E9gpDX5yp>G}qpyL@AywGP$Y^l0`UnW@pjzw3w{^X6o
zoC|`vG9;^^)Z2FP$G&BVAtpznyxq+B9g)xj{e$g^g4jEaWL4?)4Rd7Z2Dk8Z&C
zJR!si_aJ?q9)#|HKgn5vDqd$>m6G~SV8(~I*JLqCT!~opW#}r2WAdhi6|3P_g2*Ww
zZ8{4VI_99hxn12_vG@_qo%Y>BObN{B*Gl5{!72f8EmCq=-$<~O$l?=7g|sysm*;Jp
zdSz9cCZ`P6LUCLfAjG$(+SBj$sF>M9IZM7sV9l&7Qz+(tIu{eu;k<~6Ahu2$qR}6z
zjh*>w=1}KCZFvkPmRO065*hZ|8aH#~CTVfXN5J30(lB6kNb}aB_bMYHzqQ}he8qS8
z2G08B5Mf8<{Bd@oo{A-11pWhy9R^@efQhS>9HtM5lp$uPlB#RM?_!6K5soc
zxvrM7P|5Boz`w`;k<#PRHVg22o*oe!mm2*o;zj-?cTpCz2`ldu4ty|73EAPJR%!y!UKngkGkNy6Z9T1XP82uRA3TmVbZg9N%ef&_pP
zq5u&BbU_?Qp!m^%qaR=ll8^{L4^R>iMIHb%Jdq^fh$K?nAkG;^e
zv|mG7LSnztezpb_FhHW*@IFL041mO;0RyNp6fnX2P>dKr6psTOK^G*}3BWr62+#(9
zu(H&)ump51Os&i;#h{eJmR=qnc*1|UXj&qybk6}=>Lv(n0K9k((6zEe?0+Ia9EHF0
zIlu%#@$dJc1n#FBYa`SVwr1Lru>EHMBmr-bK*a2~{TDZ33NzquYLrwb0^a?g0U+u^
zB6%pnU_L%RP-ib935q8;Lp`tu{17gG7$V?3IMQS79@h*5+f?bpQq3I~WF_>VR&Na8`Rk(rqh;Eu%LNFWZ0
zLs0@rNRk&3a5%70-XL1!7eNrv^db=Udl>)eBK+0nPwEW)CM}jHjrBgOYqn
z2hscW)U}Kh0U3E&KvG&3*q>8v99k3a?oMG%gzU?wg`wy~!V~;p|9!sQaCjeF!2jWN
z!r;(O`)7dm@_$6@^dH0T6@nczv3(GdOr*|>l8((%G#O_1(?py+?q_20or
zcMR6=e~14o%o^NhEoy=%xFfOu@?nTN7+(-=h9RL`eo6ex{;QHY&KU~=5|T1dxU{U)
zFOSv!reP@~O}UmZ`xhI<5C6B1GOZ{#97rSr(lQ4wkfO`K$WsnupE>|D)zdV&WP0|$
zPtrlGHV%bHV{p!Z)OlHd07)Pq{UC74cu1W;4+Kb3W*!at9?TB_gW~WcN($iNMG6F*
z@C3;I8OzB4FoS*5FLWM&8UBsr0GRP#NL~tnS^SMCd5FJ|3d!>-2aA>Z~%t;4N^LL{01qxUVlR=O0e$%{HKV2X8ZTc!NS1*R`ov@
z?*X7-%hu4q`vyoz0uoZ^
z;rmo2DVo9q|B;CLbpal%8RhK%#QV1o0D`_C3Nk)|M=3?PlIyOQ1Zn3F-lr8(fG(G@
ztJoMM(u`*eHuG?QYURwG0aXh^8$#{~7vPQb6t4!QgyM>9gd*6md((}{?UV7b$aeLAaq(9UdDq!{4d^`DT3HwF
zKKh_ChYuPs`y8?~%je)t0zdTMEMQgVY(56-`oMTyG@CB9eU#JSHf%J$E%Z}{SDDRZ
z^L81zdI0%!u^56j{uYLEkrTfTKavoQoGk!d*u@@I@KbiHq?%9p4D2iLr+T
zT&gx`*hzGMzzIwa@@4w{d*Mft(;}#S9nOs?O&D=bI=uAdfAHqv@QdtMG8KIIK$f>>
zwTwLKe|&ZGg~f|&ji@r}=&Hwh)XEU>5(|^f@j70jvv_i?preDmv&JSyh!K8|^Q*GX
zJ1r&Wz^N!-Was49O#Snjuiv~pp?i4eV}VkZ-ZSoh7lY!r?kZ?K-<~kI3l-+Nx6-HO
ziV6lT7P}M}LNfR{Cn`15lm_L$6x4jv3~3j;*T2tW^%Q$BVdPqZiD##l{p0fXUFkL5enUeSR1gi}Tdb1`AFNhj
z;FF4f=)|W=YNd?GDR$i6%?yqk>g^vAZ?KKJ7t
z{VuaN{(Uji`kB9br7!o`%WBy@b;8o800E7EXc{gh)z;*P8mmB=zZY{q!#%4beyY54
zdZ$
z|FBRO{~1o>{fs_hTUR=T?a&KUjT1~T`t>}9{I+WZWfJc&A<{E^-=JK1e2bkA)kM3Q
zK@X{{R(0dC*%Hyuxvc4Oy{%;;Id=z2m<|qsY1Ga_=rsF$Og}zW%d5V02D)^Lflt!+
zWY~g5Ad@fTTIRNZ6PimyNNzaXIf(v$)PMzVS53)claHGm%QdS3d!525?Bdh4J`OD@
zZK4AYlPp)jj%Lc41?*RJME4(;OH;S$xvkb|E+(n=p!;~}n1qjnwQLdiQ=;ACMXn|b
zOui%1NjK3d*-|kCO-+!|o4B^5u=NJl5%Zq2hbN=cllkw@p=|ZZOxAT#Suc-&Y=woB-6;@PB3xH*4HmmJdd@?5@uL66sSdM=;Ho=d6u)p-K=SNVzBBzWqI-?a$t=|P!W
zf85j5^u2n!d#64(@ffR~=@T~lQh!@VJ@7_M=3B_mpy08zd&9@kN8cM;&1U+@(m9mL
zT*~lq_|9e4PbV*0%x5Rla0_wo$Y&|Fchl*<)XoPlcBNa48k4_1I@cP);QgJES!_tN
zu*OZw~${9oI_#$)Uf&35uUZ$*a&LYvSddV
z`{UGhE#!4K2Ahu)Ek-+MXlViu#o}ogzOo=tN7ss3!4&M$1(jjK-M}Zvy
z+ynX6sRWNg?X4((rWF~5>CsZXG<9!K>TBRQ)lmlSz&|RR`pIqeP)F>$YnXB9Nli!HCl$s&%!(#{JPky{jogb6
z_tgbo`;3zEIEJJBg;_W_peHRBq1szXy!y-O`D3xZRk3%}1SXDR56?9}sWFiI}4tB3e3X10kI$bE6p@4KtC+?8a(+
zr#FH(VzmT{i{6^V6gK#roo9g!u#EZG24yAHW#3lNd(=KXq43$JJad&t${u-|(alW-
z1*c84ZHq>K+K@|vYRE6D*AIC#$Kyh&D%*8)wbwYvav5_fC65(Hzqy|=v+Hb|Zs6|>
zyZ02gfc^5U=|Ka8rjP{p;sbZ#Vi$^-{SI2{c~L5Y|(AiEyfQGm;!VDuKK-F
z;`8lb89B?mU8;0BiNRHr;@SeA+p6=5H*?|Qm*}XJx^6li+G=QA;k*!Czhs#R&3Lca
zu4T%9b!_P6iu>fDtPXFTyTzs0-83npRPOTx67SURgm|)^IZQDr_4IVB(xgI4Tu1nZ
z`D##LWrfBHD*No2bn4LbAVj)#$rlHuHPaQX7P+?Mf&|8gsr<6+&Kh-R!yg7$Rak(-
z42Ch(Im-RHtH;0QJZ$m$7?1OOZP;)zuw^KJ;fowg#wJ_I)$?ujs?*i?4E9D+dlT<0
zw2e=4j5Zp8cob`ZsaciqE}MG8?Ne%|Gr5c~`j>`lC#FJw
zxh1X(2(1qwiq$Thw#xXap^~mNb9&`oWZ$8S9kZ%Kkd2ay*bkYH9Gzsi+LSI454km*
zNYw3Zk&G20n-*FHyS*OYWQ%=M*JOxWf6Hw_4Qy2P&CtBfd^jUyTjB6vg+C-P=_>n;
z(R7PzeT%d7vp`u6%t8Kc@;6scaSjZBQpGQPAyzl5V10CbNGK~rV0n$uB~Z=S`kH3S
zko5&Ty%Im1d?>n~s>lo%Ff>0LLF1CRnX`QyqnH(xl?u}M0y!Do{EIV1mPMWY@KP;*lZ9f{Z9pdvg;<8)`en
zV&#k)>wad>(4LL8izpxSAdVD#-O%Hz3Z_vNxt7mqpk~6q7OVJFk=$YR*o9$z{E7+h
zE?Gh?Dar>?DvlW1m@PggVOATUD=yVF82_2xnflCj*W|El==fK%N&DM>8^mir%9l#}
zq&ykc5#Ly01MIC?k*eM_7uGh7<>DSosE-^zcJBK0vLJodJmX}uJnPGamoDwq
zo8FIHh88XRKd!UKLAm;H%)>or%3g+i-BMCi97vT#-)z{J+nc%4Sgr@<2`Q?oi*G9R
zw9~Y|q9LPvOL}L7jpZVL@144r24r*813M;*#9GsKmTWr&?I(nOdF;tEEHE~%l>sUN
zb^7x`74#1gb1Igf)9PuQR`ggl$Y`Oc%r^eeyvBQ?^
z1@M_9vW$|>XelTcI8psATA(I$2kN6`|gjh
z=@Cqkqn1TY0`Hz4IuiN8&i%%W#$ss1Dc3Rh>LX{?Xh9;g@T#8onqh2}UMa_Lhp-@7
zvhz4pkzXrM2rtWjb!!wdoNzqnee?A$nU4CVar+gC*>LxcQ%*ZHxrJHO*S_n4CR23V
zHD{HwRJn4<5q>+`4NV^UYw;r%t*8d|A9ICYd+t3HQcVyE>;B`X>uJ*|I}WPA0%8tR
z_l>7ff{e;`qR$k9S{{V(8(Bcdkp|*jJdNh(jGi*<*HgcLYJ@K5#1F#0jK4H!6rW{`
z4}*5~rZJU98!iNaW|zdJ{Irby@t?9oNGeN{P1X+$tY%lEcP3(pi%MRejmXx?J*@ov
zF2K~RD-vjUt%<8EySTT_{Vw~CHRznsT+f8NSVgZivP|bFg;1+u&^n#X|E??&^2&)M9$~oTO)9Fge4Q4@o2%~=RA^#qPY!|iXKK&0
z3g~Ak6bAqwsGlDiSeK#Sc;#4OFAnLX?{f`s^co-xrFNAJ9=_AHFbPGc)SJ^!KI(I+)%~_Pfd2hlZpvu
z?(55o#}zjks@z}#_Ib#qcb~#-liwt$kI6Zt>%I&%XMcT>OT06|*m|?VM&qj9j!q0u
z1>eR6KAd@}2kj5%;SlB}EVb4Y(sWXB&&_Q?#=3yK=(EQfubdXQ*XnD%A#VMO?7CX5=pbzlGDm^c=PK
za|>~Wm85#i^j$t1F__odWhpDT6s7JvFci}*n4L$xQ@JM<=1Fz=b&bjen@jGgU5j#m
zb;rKANtfPDzZ~(*LdR?NwrAYg0>wWjSx5PHROIJl;t&zY36naD@{t6q=Th~jr`{bA
zgf}9{m|$rue;aA8AaC1OFA*}#I)fWW=|CpiV`9euHNUkmCNWZft>T^a%JpSS>6UJZ
z;K0?xn>GnG0dhU1E&0jI%J1Wr1w1Ez%_3Ac9{ddEjrZJs%aI`Kt+%aK>b&lJ-^Mw<
zy6!{u%Ldx3VV_#rn6}FJXi2i7-NnVznD+Yv`Z_%wEVd8JZjAh>YUObC>1Nw_YfgMT
z)TG96!PW7?`EGKBU{*IWWA7C=R|9eQjmexKqNSmxK3TTfA}H@?Q7}3E{@V9{Hr!C>
zErrUa7Bzm^VefO`UFV2RwHqm+kwq7PC9$^8*=KMiWqL|mC5)IBhN&ZQ%7>$5#i;fE&DpRVf
zessC*1qugFjk*fc7qX~aw!ptO$(8WW;=m2kV(-4}Y9u+|Y&NYj6zRS>4vd~~$+|`8
zuXfT%YHN@1Dgm!U-%Zhbrc_0r)rr!b7iYW%OyyM&LSjEz8KYLx1o)PYH#`$kdSW*>
z@$10?44v{kn5=}nGG?y=w=0fc)BQdlFK_!D???U}l_a^ySwy7Q?^WpI7V++eyFPb5
z%?(M9As%rDYeJg8I~n(Ve;s&ux}qA~ui~LBej~~oqWgv;
z$TzzvWqLMp3#vs}LS=#z!DWv?I!unjPRSC#B_Md@m{S>HEe!93?Iyl8-CMB2oiwvQ
zQvlspm1cVQR>GzXS5%lH#cZ;tljJi&Sc?@R(+KTE?r!ZL0iUdtHw%O&Wu+!lf~{LN
zKM|lwhHw7?ulkaFDm?R;UB~Zg+FX4(ql=3Z-4o+(ZGdCeuc1o3(!6$l_$T`w-4=6|
z@!#aYzdV`|Ys^Ps|M6(H4lvJT{^8L)MLigx8W@%g;y;I@Wo^4nV>qifW
zn*U3+X=cWjwfb)rifC*b7&^4}p~sKpzsXM~vDfkcO+wTXu3*RFf_VQ+M9}iVAcFtH
zTCxHLM(>^|RQO$#7WB?fi=W+AOV~fgA|RlkcuqkPh^^;nKOf5!!w9u{KwAE^Q<$%?
zsBQBxXUNhtN;gXMVmqI^$k6
zlPyRtJYUXT99uw|Y|VxfQ;Rl-v8_VGwzdHTGT9|6FRs%);QKKJj&5+qXnRil6GR>4
z*Zy{ZyHZXpS{}zfwv8D>6ys>Jtc^+ovZ}7fBDSi_6+xwsr4zem@+rDXX>u4g#uByE
zr^GJD9r=onG?)hz!^{-P@FY$a`+VDCn2s?1S&tBv%hsomfUd{*={z5C`Q9Qsq(=eb
zT4hE;4KlE8B${=+aB*ZxWzGa$8hSNZk~8EKed&mV$GWF_1i~m}njVrplWT55SknU@
zrJ0@$zQMv0%FV`nKa?WR05S*>j6CjOw@+1*xFkN!P$Nl~XHa--kAX-zCt}bAv+?5N
zn=@w%wM9jPg9%5ZfZTL>Jjj?gD`29X8!z8yVs$Aw>DfVXnb^sNt{Ka){~RMB@i-XD
zp^arjIk0bnh9Q`xt=pkHollrHGOW`6?l8Ff$+To`Xfd5fA4Slnow*H0UwI_+s%@?E
zqnwb})yUZLHJOksIys`~iwtT!M8)Pzq_um^)2CP}$_Bdx?{P!y>crAM$O6J%WLHLf
z!9L}ee#qPszEjJmHfO$x1r%ADU;Q`8=|H>mzzM-scm
zI_1Z=yYQ1|X1B#-&@F0By(_QUp1&AC@(C54XB~OIiQ0tDKJxZeeW_L1H{terFxk{|
zs|@kt{DOIE1zJr0!>95p(DGVCIKoe!V=90^Xd>Vz#6s;VV;0THpay7ruC(87+t_C}
zHh?pt(jnTDae%12sL5BBOBDI}d(G~%h0(akl-~5t%W_$`>AJT`lw${&`L0Qh*%4tg
z5dQp^1B(vlwFz!(`$(+-oD7XTUuH3UrCMzFRASNDI};e+4*Iiy>M%1KyZkJHxDog5
zJbwM^Pf=2$uhAV#60JbrA`G{Zxu~Qgj5&07RGDJ#1tRc+5@;s6Lw;XmibMcohN1xv
z`;)foi;v|GTIKY}wo5gC`fZmk)KSND0#s}Dy~ZpquH~jsAo5C(&r;M9l@JKC3dsv>z?uA39+)>{JIl2T7T(|+W+-lH
zwMat30zV(4Q#4R0G+1qUkX5hTvC#c8UAO`FbC|CK0#9mvO9O5iTG8&9QyNi>7Kuu@Qiy&49@XkDuyfVA6@SZpcRjlHmBjv~>PDG)%S0{mnmwGV
zjn_{59b-tJpm?WB?jCj%y44Af1{M
zU2ri%R*PQzw7xChuW`s}fk+g+Fj3MfmaGrgkJsuo22}Oc4&vHlOHuDX_(T8BGHPZRj}Sn)
z;`y3T0#j`Smx54(ZIs=X^(?QdZ9MMJ=S7q3$i2~+8ECBMxeRj-qpkiN|y#+~9Jr~aG!s{G1CMHS@G{*~h
z^u3D%&sq(CE?6YpADO+k6u-^VxqsVjKhO1zIi7{>J3}k??W575D$9(~o-{#M?c&v(
zikIPQ-^w+ahS67X6MkGgqUnzV*mf0NVM&#!jr_tu9z*lwZsGO%4P-_=#?VA1xuBz(
z%P0sQ`bc#lIC~#8XpurDAKQ|A-^P}>UdfnxaiT^2!Pajz85ajA
zs(L`EJ79D(>DA)1-(w@kldE#WhFxAOK~KYo;=o}Q?nE{FvDIMvj)@^O>!snmJ81poiAnLxP(a}M0&X{9;ncfp@q&8RC7R;W=!nD#M^QB>m
zG?WYJGMu2tN+Q?s_~xmgm3caJ04}hOqa^1^#SCuy`6`}H?T9rXyc1Qx<2x*M7P;+E
zD9TC{yfu)-2~~GK8~FsNJ`TD@e1vh08}xtTrL>s*akawe|(xtMN|Y80`zQikdGy
zXR&oWzK+m7yjM}I+`%`Rm3c9w>raN!d4GDQ?G9yqu?Mj2l*h6B7PVuKrovr8z2Zie
zH&4fBgRhV}^gb1!k3&-X^OGwMKGTiFByH`EpgS^63=iu}6puKqAj20(%KgZHo00FP
z_*~u4ZcoSNDd6noKE$I;X^dyCQ9v`Y%7HT94YmrGq#R80bEszjJEvYV=h4!|DzdDt
z?psQ$dF@^_j_R4BIxV?S;D4?ZiaP15YP#{8Z_N^FrKSe@i}Cqd?o2=nT25x!kKWsX&56TZY830Pf_~_}$gtZ@!)2vv4VwD6!l@U(A^~3m0U-3wY%ecn=oqFFk(sX5awe)~6;o`p)VU|9f%L
z9=_8z&7Ij>a3S!Q8
zWwUk#1+?_FPSPP8#ZPy)<1cP*6vKWx5X%Iz03)UT^iofs6X)@hJI~Xqd$CK$ppBYe
zNsA_8A5#K1+GswUi$7}l?GM(m1zw>$K`51bY`D3u2`t=2D9*THOEc0ZLq&Eyi`}S65>@Ui=xR%9pEZ?(g~ksKQqr
z0BSGB+nQmoZ{CA=j`^uJMOc#(g`B#f;Bjquc7_o;U*NKvKqK{8Ybn;o#95PN-AE(F
z-QU|W>)xTaoe|m|dQsoXha+9LL8;|0Y{3P^N#x87*bEomdP37Ah7oh$#2SQ+1*STR
zTPmQOIj)+6R=Y~8MWpGs+rHB@>7Z7Bz(Mwp{di{jyQVw$pwY;A%n>4Pf1&HMvX2F=
z&t7YF+8VVm3&j7%V{UgHsc~fP@IU!`w6nNa^*y4tF`{HJ*SLQn#%si$KSp@%Q*-X6
z)X}5aJH>u2hfa^2GOZk-2D`Q*XWlWE6f`3#1}d
zzuvafO!#X0r90(VZcP6fD!MC>7&&!fxmtvGP0RbI8!Ff4wI4Yf(#?Wx1XsTCly)~u
zF1v3TMVD9|4;3?t6>){gH0zxazWo_8H6YJZvfN3;
zZG6}89dGQ#Y}gKG)mnc1FndAE=`xwOxJHE0$#4AUwYz;MhYa6FSW>Z6*H8JL@OSi
zy#JXWZKQ~fHRLXlJy~AgYNXb&_BCO-dN(Wf0(6}VOA7yftx#$ZbaDL|zWm2~n4j13o%x3^BcBh#R}U3m-Rc9autQTmVj=27^Lzm=@z0P!cQ?U5r8uO06EROYLxibdrq`bjg}G!CdMME+Gf7_{sH8_>R3K
z+Lu*mRRKSH4SKX6Pa)=bY3bQIq@;ffi$ci+nEE}w=#VpRY0W&szm}>Cl!fDK;v!77
z4aW(T*W#1Lau|znRVXI1D!~J|tDluvoAe*o9hMBboi^_SvSfMspIMUBX97WulxyQ;
zL$$scBwW>h)6VUS$gSsvqejSeD{?Xm`=Qqem1vf|@2~)WANtbPImV*(gK12nK$7#D
zN+f{rJ(+Ncn7b@xy^6sAg^bOlg*{d&XEr|D!FPcCcIq6vx>6b@N`=R*JM1|5^fs2r
z>g#mcYjgq$nGy$_@UZ(?SL^PYW<4W~p9Mlsp@iLYek|_?8bkQ!ca`T?VDp*{MDsN?
z!joE!<(a4Fo_VR!gcA?tb8DLWjWq@fe`~#MNbSGKo;`o%W5i!sq91Lb)kto=bNI(c
zgw}B@SyM{REG7?m9fJHLH{5v3bk^YSXe3r#Yi2_~`SeTc$bw$G(85*8jz^Gl^|3PM
zwh)SGwPz3S4%^h8jNqT}6&#M*yu}V>gYiNkd=NN0GR&%Q~#42<97R?+)g4u_{h+28G&n
zGQv;$d07)U#D&1mqIm1x?>Sp_rtp!G5ZIdZ(I>mL(5aDxcSe~ZpM;D|f2#1jC`o?v
z*Yo?#>6q)t)3FmE-zn-KE|z>OW}@LM+#Ye$weo_h}v83y>vGbJ`+oaf*fB$?x89j>Bm)
zLZ~PGJn7=VxXr`O&F}n({xJRC^RlMN_{25&b$?n}B)TB5i^lWtDZ@{2slD%P_ZWgn
zt@ip4k7)H+jL-#@Crt^rC?KM8w2SkGx!thqI$U1#mdZCjw6YZ>O1C(3oUI>eN#+qu
zpqzwJ-D?D|URk#%$M(^wj3E3>B@qNlecu1gm#|`I4B#p)fmwJ5D~X~AJg=xMZj3>{
ztPCn*X##<*zrrY@Guw2HD(-AY{s
zTJ}c-)7^Gn+T7pvuYFA)s($hiU(WP=eZ^eWUQa9*K^sEvOt~>^d1EgXJ=i^A1-Yzq
z6Ti-8d*Q7wGQM?c1$dpDHx+6R23L1f!b6pB53zwvNbKnV?id#=D>ShjeK^Z3C>dfs
zTQIhI)!Eed$Z;N_xrZOV>L8pgDbWtKD!Yc|IGEN2JWcrjwftX_Vcvl?3Tb#V3h(=`4bHMCOzvc^^Q?}ym$(YBHY}Z
z^>MiQ>K9y6W>y!86-NGOL*69a(ZJ&U-qHAU9kC=vcl+-kE{z@>fzZe^=|9rO7AoU1
zW)40sER#L?QYT?Pczfu=93^{WD|Hc^Eu(;geYQQ`GXvi|X3+#?Lcs~}#-wcklGka#
zMw9>-{Ks6JvbERwm7HIictk-=D%Ff?F4!Q`pK4spRN~f17_Oy68CbgW*{a3xuWUpx
zZ*6v6|GMny_^l0vz|qdq-kTiJ&&H>}!st^#7;Kh^jDio_-_iIhh2e1&=~ZCr@Dl)s
zx4-U+b3d{mZ%~5e0#=)y@R?@hm;9zy?~_m=K_z?U!V?J3-X<3+y2fpa?&~$DkmnuC
z&f(Q6M=4(G+gkFvH}+-$+0qFx#D)I?sdTN9m1aD<8$8IF~%IMW9nki6)TCXEVz
zL?ZWwg1G2e*^hJ14Vt$&ViTTuIsOHbxR2*99gL?XoO2YKLfc0wgI0zJV!3;0FnNt5
z87(cXvDCHK^*%AiQI+fm>-Y260rfwM46gBIxw21Dk%p4n+*B_lNhjC%W#kqPxgN}`D7u|d@wjfQb2|mCL;}n@IqvHB&4KdMQQ*4LqHE3
z{e$(Vg$eS|{-Y&K34Kl~YSbYI2mXs!j)X5FCmvLtW#U_XLE+ex*SNM;HAh!SdYPs5
zSzOq(yQwKdc@4On7zlXGj;k%5BS@D$G_=v@Gn5d2B3iEy#|VruszIlT^pl852W~Gj
zKK+bCqB3}iITYA#p_qd2e+EP-hH-@d?0IC4YG;~UR{$!N${3P(v~ls0ENE_pX$-KE
zD!b&_+g`l}u?blI8g0jJ$;HytLmbVD>3ov^?y_!U*rY9NOoQzv$+YM{few^LRct!TARrpLd`SjPs
z+vN=nRX;Ma+xr}U_NwM%K9i5XqcX6H55EqLeX<<9-+5WHI|5f_NNR50upO@;$`pyx
zJ3ouhcVQo3`)e)T8tcz+D|s~=TGWy?Qm_hmx9Xyl&Ye}Na17b)77ijm#|9yVq!pw<}~&`#3l+@!>E&I35#|jH)c&{{cOBxyAqh

diff --git a/doc/src/DecisionTrees/DecisionTrees.do.txt b/doc/src/DecisionTrees/DecisionTrees.do.txt
index 3d35629da..8926c0e71 100644
--- a/doc/src/DecisionTrees/DecisionTrees.do.txt
+++ b/doc/src/DecisionTrees/DecisionTrees.do.txt
@@ -466,11 +466,12 @@ os.system(cmd)
 
 !split
 ===== Algorithms for Setting up Decision Trees =====
+
 Two algorithms stand out in the set up of decision trees:
 o The CART (Classification And Regression Tree) algorithm for both classification and regression
 o The ID3 algorithm based on the computation of the information gain for classification
 
-We discuss both algorithms with applications here. The popular library -Scikit-Learn_ uses the CART algorithm. For classification problems you can use either the _gini_ index or the _entropy_ to split a tree in two branches.
+We discuss both algorithms with applications here. The popular library _Scikit-Learn_ uses the CART algorithm. For classification problems you can use either the _gini_ index or the _entropy_ to split a tree in two branches.
 
 !split
 ===== The CART algorithm for Classification =====
@@ -1148,7 +1149,7 @@ However, by aggregating many decision trees, using methods like bagging, random
 
 
 !split
-===== From a Single Tree to Many Trees, Meet the Jungle of Methods =====
+===== Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods =====
 
 As stated above and seen in many of the examples discussed here about
 a single decision tree, we often end up overfitting our training
@@ -1160,7 +1161,7 @@ machine learning algorithms or just use one of them to construct forests and jun
 o Voting classifiers
 o Bagging and Pasting
 o Random forests
-o Boosting methods
+o Boosting methods, from adaptive to Extreme Gradient Boosting (XGBoost)
 
 We discuss these methods here.
 
@@ -1389,6 +1390,66 @@ plt.show()
 !ec
 
 
+!split
+===== Making our own Bagging with Bootstrap =====
+!bc pycod
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
+
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+    model = DecisionTreeRegressor(max_depth=5) 
+    y_pred = np.empty((y_test.shape[0], n_boostraps))
+    for i in range(n_boostraps):
+        x_, y_ = resample(X_train_scaled, y_train)
+        model.fit(x_, y_)
+        y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+    polydegree[degree] = degree
+    error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+    bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+    variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+    print('Polynomial degree:', degree)
+    print('Error:', error[degree])
+    print('Bias^2:', bias[degree])
+    print('Var:', variance[degree])
+    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+!ec
+
+
+
 !split
 ===== Random forests =====
 
@@ -1505,7 +1566,6 @@ print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Ran
 import scikitplot as skplt
 y_pred = Random_Forest_model.predict(X_test_scaled)
 skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
-#
 plt.show()
 y_probas = Random_Forest_model.predict_proba(X_test_scaled)
 skplt.metrics.plot_roc(y_test, y_probas)
@@ -1539,9 +1599,66 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
 
 
 !split
-===== Feature Importance =====
+===== Bootstrap with Random Forests Instead of a Single Tree =====
+
+!bc pycod
+
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.ensemble import RandomForestRegressor
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+    model = RandomForestRegressor()
+    y_pred = np.empty((y_test.shape[0], n_boostraps))
+    for i in range(n_boostraps):
+        x_, y_ = resample(X_train_scaled, y_train)
+        model.fit(x_, y_)
+        y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+    polydegree[degree] = degree
+    error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+    bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+    variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+    print('Polynomial degree:', degree)
+    print('Error:', error[degree])
+    print('Bias^2:', bias[degree])
+    print('Var:', variance[degree])
+    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
+!ec
 
-Example will be added here.
 
 
 !split
@@ -1601,7 +1718,7 @@ individually at each steps. The obersvations which were misclassified
 at iteration $m-1$ have a weight which is larger than those which were
 classified properly. As this proceeds, the observations which were
 difficult to classifiy correctly are given a larger influence. Each
-new classificatio step $m$ is then forced to concentrate on those
+new classification step $m$ is then forced to concentrate on those
 observations that are missed in the previous iterations.
 
 !split
diff --git a/doc/src/DecisionTrees/Programs/bootstrap.py b/doc/src/DecisionTrees/Programs/bootstrap.py
new file mode 100644
index 000000000..4f6dd3ae8
--- /dev/null
+++ b/doc/src/DecisionTrees/Programs/bootstrap.py
@@ -0,0 +1,56 @@
+
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
+
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+    model = DecisionTreeRegressor(max_depth=5) 
+    y_pred = np.empty((y_test.shape[0], n_boostraps))
+    for i in range(n_boostraps):
+        x_, y_ = resample(X_train_scaled, y_train)
+        model.fit(x_, y_)
+        y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+    polydegree[degree] = degree
+    error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+    bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+    variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+    print('Polynomial degree:', degree)
+    print('Error:', error[degree])
+    print('Bias^2:', bias[degree])
+    print('Var:', variance[degree])
+    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
diff --git a/doc/src/DecisionTrees/Programs/rfbootstrap.py b/doc/src/DecisionTrees/Programs/rfbootstrap.py
new file mode 100644
index 000000000..824862286
--- /dev/null
+++ b/doc/src/DecisionTrees/Programs/rfbootstrap.py
@@ -0,0 +1,54 @@
+
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.ensemble import RandomForestRegressor
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+    model = RandomForestRegressor()
+    y_pred = np.empty((y_test.shape[0], n_boostraps))
+    for i in range(n_boostraps):
+        x_, y_ = resample(X_train_scaled, y_train)
+        model.fit(x_, y_)
+        y_pred[:, i] = model.predict(X_test_scaled)#.ravel()
+
+    polydegree[degree] = degree
+    error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+    bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+    variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+    print('Polynomial degree:', degree)
+    print('Error:', error[degree])
+    print('Bias^2:', bias[degree])
+    print('Var:', variance[degree])
+    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+