From 15cb23459d3c6df42f358d8cb77c947081e1c040 Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Fri, 4 Nov 2022 07:37:39 +0100 Subject: [PATCH] added video link --- doc/pub/week44/html/._week44-bs000.html | 47 +- doc/pub/week44/html/._week44-bs001.html | 45 +- doc/pub/week44/html/._week44-bs002.html | 45 +- doc/pub/week44/html/._week44-bs003.html | 45 +- doc/pub/week44/html/._week44-bs004.html | 45 +- doc/pub/week44/html/._week44-bs005.html | 45 +- doc/pub/week44/html/._week44-bs006.html | 45 +- doc/pub/week44/html/._week44-bs007.html | 45 +- doc/pub/week44/html/._week44-bs008.html | 45 +- doc/pub/week44/html/._week44-bs009.html | 45 +- doc/pub/week44/html/._week44-bs010.html | 45 +- doc/pub/week44/html/._week44-bs011.html | 45 +- doc/pub/week44/html/._week44-bs012.html | 45 +- doc/pub/week44/html/._week44-bs013.html | 45 +- doc/pub/week44/html/._week44-bs014.html | 45 +- doc/pub/week44/html/._week44-bs015.html | 45 +- doc/pub/week44/html/._week44-bs016.html | 45 +- doc/pub/week44/html/._week44-bs017.html | 45 +- doc/pub/week44/html/._week44-bs018.html | 45 +- doc/pub/week44/html/._week44-bs019.html | 45 +- doc/pub/week44/html/._week44-bs020.html | 45 +- doc/pub/week44/html/._week44-bs021.html | 45 +- doc/pub/week44/html/._week44-bs022.html | 45 +- doc/pub/week44/html/._week44-bs023.html | 45 +- doc/pub/week44/html/._week44-bs024.html | 45 +- doc/pub/week44/html/._week44-bs025.html | 45 +- doc/pub/week44/html/._week44-bs026.html | 45 +- doc/pub/week44/html/._week44-bs027.html | 45 +- doc/pub/week44/html/._week44-bs028.html | 45 +- doc/pub/week44/html/._week44-bs029.html | 45 +- doc/pub/week44/html/._week44-bs030.html | 45 +- doc/pub/week44/html/._week44-bs031.html | 45 +- doc/pub/week44/html/._week44-bs032.html | 45 +- doc/pub/week44/html/._week44-bs033.html | 45 +- doc/pub/week44/html/._week44-bs034.html | 45 +- doc/pub/week44/html/._week44-bs035.html | 45 +- doc/pub/week44/html/._week44-bs036.html | 45 +- doc/pub/week44/html/._week44-bs037.html | 45 +- doc/pub/week44/html/._week44-bs038.html | 45 +- doc/pub/week44/html/._week44-bs039.html | 45 +- doc/pub/week44/html/._week44-bs040.html | 45 +- doc/pub/week44/html/._week44-bs041.html | 45 +- doc/pub/week44/html/._week44-bs042.html | 45 +- doc/pub/week44/html/._week44-bs043.html | 45 +- doc/pub/week44/html/._week44-bs044.html | 45 +- doc/pub/week44/html/._week44-bs045.html | 45 +- doc/pub/week44/html/._week44-bs046.html | 45 +- doc/pub/week44/html/._week44-bs047.html | 45 +- doc/pub/week44/html/._week44-bs048.html | 45 +- doc/pub/week44/html/._week44-bs049.html | 46 +- doc/pub/week44/html/._week44-bs050.html | 47 +- doc/pub/week44/html/._week44-bs051.html | 48 +- doc/pub/week44/html/._week44-bs052.html | 49 +- doc/pub/week44/html/._week44-bs053.html | 50 +- doc/pub/week44/html/._week44-bs054.html | 51 +- doc/pub/week44/html/._week44-bs055.html | 52 +- doc/pub/week44/html/._week44-bs056.html | 53 +- doc/pub/week44/html/._week44-bs057.html | 55 +- doc/pub/week44/html/._week44-bs058.html | 229 ++--- doc/pub/week44/html/._week44-bs059.html | 262 ++--- doc/pub/week44/html/._week44-bs060.html | 226 +++-- doc/pub/week44/html/._week44-bs061.html | 315 +++--- doc/pub/week44/html/._week44-bs062.html | 277 +++--- doc/pub/week44/html/._week44-bs063.html | 277 +++--- doc/pub/week44/html/week44-bs.html | 47 +- doc/pub/week44/html/week44-reveal.html | 387 +++++++- doc/pub/week44/html/week44-solarized.html | 359 ++++++- doc/pub/week44/html/week44.html | 359 ++++++- doc/pub/week44/ipynb/ipynb-week44-src.tar.gz | Bin 294283 -> 294283 bytes doc/pub/week44/ipynb/week44.ipynb | 980 ++++++++++++++++--- doc/src/week44/week44.do.txt | 325 ++++++ 71 files changed, 5683 insertions(+), 1018 deletions(-) diff --git a/doc/pub/week44/html/._week44-bs000.html b/doc/pub/week44/html/._week44-bs000.html index 3e2d78be7..b05de5cdb 100644 --- a/doc/pub/week44/html/._week44-bs000.html +++ b/doc/pub/week44/html/._week44-bs000.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -318,7 +359,7 @@ MathJax.Hub.Config({
    -

    Nov 3, 2022

    +

    Nov 4, 2022


    @@ -343,7 +384,7 @@ MathJax.Hub.Config({
  • 9
  • 10
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs001.html b/doc/pub/week44/html/._week44-bs001.html index 49e8126f7..20bf56a32 100644 --- a/doc/pub/week44/html/._week44-bs001.html +++ b/doc/pub/week44/html/._week44-bs001.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -346,7 +387,7 @@ MathJax.Hub.Config({
  • 10
  • 11
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs002.html b/doc/pub/week44/html/._week44-bs002.html index 4477ae9b8..cdb40b9c3 100644 --- a/doc/pub/week44/html/._week44-bs002.html +++ b/doc/pub/week44/html/._week44-bs002.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -328,7 +369,7 @@ accelerate scientific discovery.
  • 11
  • 12
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs003.html b/doc/pub/week44/html/._week44-bs003.html index 3311f44f9..38eb635b3 100644 --- a/doc/pub/week44/html/._week44-bs003.html +++ b/doc/pub/week44/html/._week44-bs003.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -345,7 +386,7 @@ where we then finally end up in so called leaf nodes.
  • 12
  • 13
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs004.html b/doc/pub/week44/html/._week44-bs004.html index 52b9eb0d5..79a3df3c0 100644 --- a/doc/pub/week44/html/._week44-bs004.html +++ b/doc/pub/week44/html/._week44-bs004.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -333,7 +374,7 @@ given some assumptions, make predictions about the target feature value
  • 13
  • 14
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs005.html b/doc/pub/week44/html/._week44-bs005.html index bcac8c24b..aa687370e 100644 --- a/doc/pub/week44/html/._week44-bs005.html +++ b/doc/pub/week44/html/._week44-bs005.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -326,7 +367,7 @@ MathJax.Hub.Config({
  • 14
  • 15
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs006.html b/doc/pub/week44/html/._week44-bs006.html index 37dedc421..6e711b4b0 100644 --- a/doc/pub/week44/html/._week44-bs006.html +++ b/doc/pub/week44/html/._week44-bs006.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -326,7 +367,7 @@ MathJax.Hub.Config({
  • 15
  • 16
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs007.html b/doc/pub/week44/html/._week44-bs007.html index e84d6c684..9b5b2bb86 100644 --- a/doc/pub/week44/html/._week44-bs007.html +++ b/doc/pub/week44/html/._week44-bs007.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -332,7 +373,7 @@ MathJax.Hub.Config({
  • 16
  • 17
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs008.html b/doc/pub/week44/html/._week44-bs008.html index 4bded7cbd..7c37a5a7d 100644 --- a/doc/pub/week44/html/._week44-bs008.html +++ b/doc/pub/week44/html/._week44-bs008.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -337,7 +378,7 @@ node.
  • 17
  • 18
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs009.html b/doc/pub/week44/html/._week44-bs009.html index 631dc2e94..f134ca0a0 100644 --- a/doc/pub/week44/html/._week44-bs009.html +++ b/doc/pub/week44/html/._week44-bs009.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -338,7 +379,7 @@ predicting the target features of query instances is as follows:
  • 18
  • 19
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs010.html b/doc/pub/week44/html/._week44-bs010.html index 614f93521..488b21faf 100644 --- a/doc/pub/week44/html/._week44-bs010.html +++ b/doc/pub/week44/html/._week44-bs010.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -437,7 +478,7 @@ plt.show()
  • 19
  • 20
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs011.html b/doc/pub/week44/html/._week44-bs011.html index 758151b73..5a1ec5afe 100644 --- a/doc/pub/week44/html/._week44-bs011.html +++ b/doc/pub/week44/html/._week44-bs011.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -348,7 +389,7 @@ within box \( j \).
  • 20
  • 21
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs012.html b/doc/pub/week44/html/._week44-bs012.html index 2ebdee727..5deb07bfc 100644 --- a/doc/pub/week44/html/._week44-bs012.html +++ b/doc/pub/week44/html/._week44-bs012.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -341,7 +382,7 @@ better tree in some future step.
  • 21
  • 22
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs013.html b/doc/pub/week44/html/._week44-bs013.html index 6bde52ca4..139ac6bb1 100644 --- a/doc/pub/week44/html/._week44-bs013.html +++ b/doc/pub/week44/html/._week44-bs013.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -373,7 +414,7 @@ region contains more than five observations.
  • 22
  • 23
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs014.html b/doc/pub/week44/html/._week44-bs014.html index b3e52518e..aea6b187a 100644 --- a/doc/pub/week44/html/._week44-bs014.html +++ b/doc/pub/week44/html/._week44-bs014.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -343,7 +384,7 @@ parameter \( \alpha \).
  • 23
  • 24
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs015.html b/doc/pub/week44/html/._week44-bs015.html index fddf9b13b..1a8365774 100644 --- a/doc/pub/week44/html/._week44-bs015.html +++ b/doc/pub/week44/html/._week44-bs015.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -355,7 +396,7 @@ subtree corresponding to \( \alpha \).
  • 24
  • 25
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs016.html b/doc/pub/week44/html/._week44-bs016.html index 6bc64f62c..b0e2116dd 100644 --- a/doc/pub/week44/html/._week44-bs016.html +++ b/doc/pub/week44/html/._week44-bs016.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -346,7 +387,7 @@ MathJax.Hub.Config({
  • 25
  • 26
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs017.html b/doc/pub/week44/html/._week44-bs017.html index 998b21aef..e89c9f96a 100644 --- a/doc/pub/week44/html/._week44-bs017.html +++ b/doc/pub/week44/html/._week44-bs017.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -341,7 +382,7 @@ fall into that region.
  • 26
  • 27
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs018.html b/doc/pub/week44/html/._week44-bs018.html index 4f688e2a0..5c786435d 100644 --- a/doc/pub/week44/html/._week44-bs018.html +++ b/doc/pub/week44/html/._week44-bs018.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -346,7 +387,7 @@ than is the classification error rate.
  • 27
  • 28
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs019.html b/doc/pub/week44/html/._week44-bs019.html index 28f104054..c708a0a39 100644 --- a/doc/pub/week44/html/._week44-bs019.html +++ b/doc/pub/week44/html/._week44-bs019.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -368,7 +409,7 @@ $$
  • 28
  • 29
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs020.html b/doc/pub/week44/html/._week44-bs020.html index d1ed27390..d30213f0d 100644 --- a/doc/pub/week44/html/._week44-bs020.html +++ b/doc/pub/week44/html/._week44-bs020.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -382,7 +423,7 @@ os.system(cmd)
  • 29
  • 30
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs021.html b/doc/pub/week44/html/._week44-bs021.html index b6d105c1e..108a6f401 100644 --- a/doc/pub/week44/html/._week44-bs021.html +++ b/doc/pub/week44/html/._week44-bs021.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -373,7 +414,7 @@ os.system(cmd)
  • 30
  • 31
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs022.html b/doc/pub/week44/html/._week44-bs022.html index e47cb9224..fa791464b 100644 --- a/doc/pub/week44/html/._week44-bs022.html +++ b/doc/pub/week44/html/._week44-bs022.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -359,7 +400,7 @@ tree.plot_tree(tree_clf)
  • 31
  • 32
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs023.html b/doc/pub/week44/html/._week44-bs023.html index a95285b90..151bda561 100644 --- a/doc/pub/week44/html/._week44-bs023.html +++ b/doc/pub/week44/html/._week44-bs023.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -362,7 +403,7 @@ r = export_text(decision_tree, feature_names
  • 32
  • 33
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs024.html b/doc/pub/week44/html/._week44-bs024.html index 6c72987b0..4bdceed64 100644 --- a/doc/pub/week44/html/._week44-bs024.html +++ b/doc/pub/week44/html/._week44-bs024.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -338,7 +379,7 @@ in two branches.
  • 33
  • 34
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs025.html b/doc/pub/week44/html/._week44-bs025.html index e5c0a7e97..381e9a73e 100644 --- a/doc/pub/week44/html/._week44-bs025.html +++ b/doc/pub/week44/html/._week44-bs025.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -350,7 +391,7 @@ hyperparameters control additional stopping conditions such as the \( min\_sampl
  • 34
  • 35
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs026.html b/doc/pub/week44/html/._week44-bs026.html index 9ee0e13a5..028ce8cb3 100644 --- a/doc/pub/week44/html/._week44-bs026.html +++ b/doc/pub/week44/html/._week44-bs026.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -350,7 +391,7 @@ just like for classification tasks, is prone to overfitting.
  • 35
  • 36
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs027.html b/doc/pub/week44/html/._week44-bs027.html index 17e91f270..5120f9195 100644 --- a/doc/pub/week44/html/._week44-bs027.html +++ b/doc/pub/week44/html/._week44-bs027.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -333,7 +374,7 @@ achieved by a series of binary split and this is normally preferred.
  • 36
  • 37
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs028.html b/doc/pub/week44/html/._week44-bs028.html index 08dede822..f5ce5c91e 100644 --- a/doc/pub/week44/html/._week44-bs028.html +++ b/doc/pub/week44/html/._week44-bs028.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -342,7 +383,7 @@ recently has gotten grades below average or above.
  • 37
  • 38
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs029.html b/doc/pub/week44/html/._week44-bs029.html index a70bfaf4d..071f0a46e 100644 --- a/doc/pub/week44/html/._week44-bs029.html +++ b/doc/pub/week44/html/._week44-bs029.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -349,7 +390,7 @@ MathJax.Hub.Config({
  • 38
  • 39
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs030.html b/doc/pub/week44/html/._week44-bs030.html index 3ac920772..235acd715 100644 --- a/doc/pub/week44/html/._week44-bs030.html +++ b/doc/pub/week44/html/._week44-bs030.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -339,7 +380,7 @@ these binary classes, they can easily be split into ones and zeros.
  • 39
  • 40
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs031.html b/doc/pub/week44/html/._week44-bs031.html index 1d402a085..23f36027c 100644 --- a/doc/pub/week44/html/._week44-bs031.html +++ b/doc/pub/week44/html/._week44-bs031.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -335,7 +376,7 @@ MathJax.Hub.Config({
  • 40
  • 41
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs032.html b/doc/pub/week44/html/._week44-bs032.html index 71ea0b430..e58779c63 100644 --- a/doc/pub/week44/html/._week44-bs032.html +++ b/doc/pub/week44/html/._week44-bs032.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -337,7 +378,7 @@ MathJax.Hub.Config({
  • 41
  • 42
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs033.html b/doc/pub/week44/html/._week44-bs033.html index 79e85f2e5..d9eaa77b4 100644 --- a/doc/pub/week44/html/._week44-bs033.html +++ b/doc/pub/week44/html/._week44-bs033.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -410,7 +451,7 @@ os.system(cmd)
  • 42
  • 43
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs034.html b/doc/pub/week44/html/._week44-bs034.html index d2315af95..84ccc6137 100644 --- a/doc/pub/week44/html/._week44-bs034.html +++ b/doc/pub/week44/html/._week44-bs034.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -366,7 +407,7 @@ humidity and weak and strong for wind.
  • 43
  • 44
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs035.html b/doc/pub/week44/html/._week44-bs035.html index 633f1ff0d..1af275757 100644 --- a/doc/pub/week44/html/._week44-bs035.html +++ b/doc/pub/week44/html/._week44-bs035.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -417,7 +458,7 @@ os.system(cmd)
  • 44
  • 45
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs036.html b/doc/pub/week44/html/._week44-bs036.html index 358c4caf2..71e9b2898 100644 --- a/doc/pub/week44/html/._week44-bs036.html +++ b/doc/pub/week44/html/._week44-bs036.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -418,7 +459,7 @@ split = get_split(dataset)
  • 45
  • 46
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs037.html b/doc/pub/week44/html/._week44-bs037.html index 108d1d79b..3daad0536 100644 --- a/doc/pub/week44/html/._week44-bs037.html +++ b/doc/pub/week44/html/._week44-bs037.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -413,7 +454,7 @@ plt.show()
  • 46
  • 47
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs038.html b/doc/pub/week44/html/._week44-bs038.html index 498154862..764c595b6 100644 --- a/doc/pub/week44/html/._week44-bs038.html +++ b/doc/pub/week44/html/._week44-bs038.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -369,7 +410,7 @@ plt.show()
  • 47
  • 48
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs039.html b/doc/pub/week44/html/._week44-bs039.html index 720347f7c..a9b184b84 100644 --- a/doc/pub/week44/html/._week44-bs039.html +++ b/doc/pub/week44/html/._week44-bs039.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -378,7 +419,7 @@ tree_reg.fit(X, y)
  • 48
  • 49
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs040.html b/doc/pub/week44/html/._week44-bs040.html index 8cb49e56d..efafcb410 100644 --- a/doc/pub/week44/html/._week44-bs040.html +++ b/doc/pub/week44/html/._week44-bs040.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -434,7 +475,7 @@ plt.show()
  • 49
  • 50
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs041.html b/doc/pub/week44/html/._week44-bs041.html index dd237bf01..422b89bd9 100644 --- a/doc/pub/week44/html/._week44-bs041.html +++ b/doc/pub/week44/html/._week44-bs041.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -336,7 +377,7 @@ MathJax.Hub.Config({
  • 50
  • 51
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs042.html b/doc/pub/week44/html/._week44-bs042.html index 6006d9c27..d777f34ee 100644 --- a/doc/pub/week44/html/._week44-bs042.html +++ b/doc/pub/week44/html/._week44-bs042.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -341,7 +382,7 @@ trees can be substantially improved.
  • 51
  • 52
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs043.html b/doc/pub/week44/html/._week44-bs043.html index 25ceb5a79..ce0bda5e3 100644 --- a/doc/pub/week44/html/._week44-bs043.html +++ b/doc/pub/week44/html/._week44-bs043.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -348,7 +389,7 @@ try to explain here. These are
  • 52
  • 53
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs044.html b/doc/pub/week44/html/._week44-bs044.html index a782cb84f..6d9557cd8 100644 --- a/doc/pub/week44/html/._week44-bs044.html +++ b/doc/pub/week44/html/._week44-bs044.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -333,7 +374,7 @@ MathJax.Hub.Config({
  • 53
  • 54
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs045.html b/doc/pub/week44/html/._week44-bs045.html index 5ebecd51a..fcf601e22 100644 --- a/doc/pub/week44/html/._week44-bs045.html +++ b/doc/pub/week44/html/._week44-bs045.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -344,7 +385,7 @@ each iteration.
  • 54
  • 55
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs046.html b/doc/pub/week44/html/._week44-bs046.html index fd8130390..fd50cdfca 100644 --- a/doc/pub/week44/html/._week44-bs046.html +++ b/doc/pub/week44/html/._week44-bs046.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -352,7 +393,7 @@ numbers kicking in.
  • 55
  • 56
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs047.html b/doc/pub/week44/html/._week44-bs047.html index 59788979a..2f81a1922 100644 --- a/doc/pub/week44/html/._week44-bs047.html +++ b/doc/pub/week44/html/._week44-bs047.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -387,7 +428,7 @@ DATA_ID = "
  • 56
  • 57
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs048.html b/doc/pub/week44/html/._week44-bs048.html index fd1e6be97..40141c89a 100644 --- a/doc/pub/week44/html/._week44-bs048.html +++ b/doc/pub/week44/html/._week44-bs048.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -370,6 +411,8 @@ plt.show()
  • 56
  • 57
  • 58
    • +
  • ...
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs049.html b/doc/pub/week44/html/._week44-bs049.html index 1a803751b..4b87dd632 100644 --- a/doc/pub/week44/html/._week44-bs049.html +++ b/doc/pub/week44/html/._week44-bs049.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -391,6 +432,9 @@ voting_clf.fit(X_train, y_train)
  • 56
  • 57
  • 58
    • +
  • 59
    • +
  • ...
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs050.html b/doc/pub/week44/html/._week44-bs050.html index e6c30d017..431a07bba 100644 --- a/doc/pub/week44/html/._week44-bs050.html +++ b/doc/pub/week44/html/._week44-bs050.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -442,6 +483,10 @@ voting_clf.fit(X_train, y_train)
  • 56
  • 57
  • 58
    • +
  • 59
    • +
  • 60
    • +
  • ...
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs051.html b/doc/pub/week44/html/._week44-bs051.html index f390645b6..afe291357 100644 --- a/doc/pub/week44/html/._week44-bs051.html +++ b/doc/pub/week44/html/._week44-bs051.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -337,6 +378,11 @@ learning method.
  • 56
  • 57
  • 58
    • +
  • 59
    • +
  • 60
    • +
  • 61
    • +
  • ...
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs052.html b/doc/pub/week44/html/._week44-bs052.html index 261eed7d8..db6918106 100644 --- a/doc/pub/week44/html/._week44-bs052.html +++ b/doc/pub/week44/html/._week44-bs052.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -346,6 +387,12 @@ predictor, averaged over all \( B \) trees.
  • 56
  • 57
  • 58
    • +
  • 59
    • +
  • 60
    • +
  • 61
    • +
  • 62
    • +
  • ...
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs053.html b/doc/pub/week44/html/._week44-bs053.html index ad7480cb2..810799dfc 100644 --- a/doc/pub/week44/html/._week44-bs053.html +++ b/doc/pub/week44/html/._week44-bs053.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -404,6 +445,13 @@ plt.show()
  • 56
  • 57
  • 58
    • +
  • 59
    • +
  • 60
    • +
  • 61
    • +
  • 62
    • +
  • 63
    • +
  • ...
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs054.html b/doc/pub/week44/html/._week44-bs054.html index a8f186e32..34bc0409d 100644 --- a/doc/pub/week44/html/._week44-bs054.html +++ b/doc/pub/week44/html/._week44-bs054.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -359,6 +400,14 @@ this setting.
  • 56
  • 57
  • 58
    • +
  • 59
    • +
  • 60
    • +
  • 61
    • +
  • 62
    • +
  • 63
    • +
  • 64
    • +
  • ...
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs055.html b/doc/pub/week44/html/._week44-bs055.html index 282032e0e..ba6f1b9c3 100644 --- a/doc/pub/week44/html/._week44-bs055.html +++ b/doc/pub/week44/html/._week44-bs055.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -334,6 +375,15 @@ MathJax.Hub.Config({
  • 56
  • 57
  • 58
    • +
  • 59
    • +
  • 60
    • +
  • 61
    • +
  • 62
    • +
  • 63
    • +
  • 64
    • +
  • 65
    • +
  • ...
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs056.html b/doc/pub/week44/html/._week44-bs056.html index 655fa726d..a944675ce 100644 --- a/doc/pub/week44/html/._week44-bs056.html +++ b/doc/pub/week44/html/._week44-bs056.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -410,6 +451,16 @@ discrimination threshold is varied. It plots the true positive rate against the
  • 56
  • 57
  • 58
    • +
  • 59
    • +
  • 60
    • +
  • 61
    • +
  • 62
    • +
  • 63
    • +
  • 64
    • +
  • 65
    • +
  • 66
    • +
  • ...
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs057.html b/doc/pub/week44/html/._week44-bs057.html index a9ca9e066..c14053e11 100644 --- a/doc/pub/week44/html/._week44-bs057.html +++ b/doc/pub/week44/html/._week44-bs057.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -368,6 +409,18 @@ np.sum(y_pred =
  • 56
  • 57
  • 58
    • +
  • 59
    • +
  • 60
    • +
  • 61
    • +
  • 62
    • +
  • 63
    • +
  • 64
    • +
  • 65
    • +
  • 66
    • +
  • 67
    • +
  • ...
    • +
  • 68
    • +
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs058.html b/doc/pub/week44/html/._week44-bs058.html index f8c397c4d..3b6bc24f3 100644 --- a/doc/pub/week44/html/._week44-bs058.html +++ b/doc/pub/week44/html/._week44-bs058.html @@ -38,10 +38,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Overview of week 44', 2, None, 'overview-of-week-44'), ('Digression First', 2, None, 'digression-first'), - ('A short Discussion of Project 2', - 2, - None, - 'a-short-discussion-of-project-2'), ('Decision trees, overarching aims', 2, None, @@ -201,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -238,62 +265,71 @@ MathJax.Hub.Config({ @@ -305,58 +341,19 @@ MathJax.Hub.Config({

     

     

     

    -

    Compare Bagging on Trees with Random Forests

    +

    Boosting, a Bird's Eye View

    - -
    -
    -
    -
    -
    - 
    bag_clf = BaggingClassifier(
-    DecisionTreeClassifier(splitter="random", max_leaf_nodes=16, random_state=42),
-    n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1, random_state=42)
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -
    -
    -
    -
    -
    - 
    bag_clf.fit(X_train, y_train)
-y_pred = bag_clf.predict(X_test)
-from sklearn.ensemble import RandomForestClassifier
-rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)
-rnd_clf.fit(X_train, y_train)
-y_pred_rf = rnd_clf.predict(X_test)
-np.sum(y_pred == y_pred_rf) / len(y_pred) 
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +

    The basic idea is to combine weak classifiers in order to create a good +classifier. With a weak classifier we often intend a classifier which +produces results which are only slightly better than we would get by +random guesses. +

    +

    This is done by applying in an iterative way a weak (or a standard +classifier like decision trees) to modify the data. In each iteration +we emphasize those observations which are misclassified by weighting +them with a factor. +

    @@ -373,6 +370,16 @@ np.sum(y_pred =

  • 57
  • 58
  • 59
    • +
  • 60
    • +
  • 61
    • +
  • 62
    • +
  • 63
    • +
  • 64
    • +
  • 65
    • +
  • 66
    • +
  • 67
    • +
  • 68
    • +
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs059.html b/doc/pub/week44/html/._week44-bs059.html index 013b22d23..d5e0adef8 100644 --- a/doc/pub/week44/html/._week44-bs059.html +++ b/doc/pub/week44/html/._week44-bs059.html @@ -38,10 +38,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Overview of week 44', 2, None, 'overview-of-week-44'), ('Digression First', 2, None, 'digression-first'), - ('A short Discussion of Project 2', - 2, - None, - 'a-short-discussion-of-project-2'), ('Decision trees, overarching aims', 2, None, @@ -149,10 +145,10 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'computing-the-gini-factor'), - ('Another example, the moons again', + ('Another example, the moons', 2, None, - 'another-example-the-moons-again'), + 'another-example-the-moons'), ('Playing around with regions', 2, None, @@ -173,26 +169,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -205,6 +181,13 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d None, 'using-the-voting-classifier'), ('Voting and Bagging', 2, None, 'voting-and-bagging'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Random forests', 2, None, 'random-forests'), ('Random Forest Algorithm', 2, None, 'random-forest-algorithm'), ('Random Forests Compared with other Methods on the Cancer Data', @@ -214,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -251,66 +265,71 @@ MathJax.Hub.Config({ @@ -322,47 +341,53 @@ MathJax.Hub.Config({

     

     

     

    -

    Random forests

    +

    What is boosting? Additive Modelling/Iterative Fitting

    -

    Random forests provide an improvement over bagged trees by way of a -small tweak that decorrelates the trees. +

    Boosting is a way of fitting an additive expansion in a set of +elementary basis functions like for example some simple polynomials. +Assume for example that we have a function +

    +$$ +f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m), +$$ + +

    where \( \beta_m \) are the expansion parameters to be determined in a +minimization process and \( b(x;\gamma_m) \) are some simple functions of +the multivariable parameter \( x \) which is characterized by the +parameters \( \gamma_m \).

    -

    As in bagging, we build a -number of decision trees on bootstrapped training samples. But when -building these decision trees, each time a split in a tree is -considered, a random sample of \( m \) predictors is chosen as split -candidates from the full set of \( p \) predictors. The split is allowed to -use only one of those \( m \) predictors. -

    - -

    A fresh sample of \( m \) predictors is -taken at each split, and typically we choose +

    As an example, consider the Sigmoid function we used in logistic +regression. In that case, we can translate the function +\( b(x;\gamma_m) \) into the Sigmoid function

    $$ -m\approx \sqrt{p}. +\sigma(t) = \frac{1}{1+\exp{(-t)}}, $$ -

    In building a random forest, at -each split in the tree, the algorithm is not even allowed to consider -a majority of the available predictors. +

    where \( t=\gamma_0+\gamma_1 x \) and the parameters \( \gamma_0 \) and +\( \gamma_1 \) were determined by the Logistic Regression fitting +algorithm.

    -

    The reason for this is rather clever. Suppose that there is one very -strong predictor in the data set, along with a number of other -moderately strong predictors. Then in the collection of bagged -variable importance random forest trees, most or all of the trees will -use this strong predictor in the top split. Consequently, all of the -bagged trees will look quite similar to each other. Hence the -predictions from the bagged trees will be highly correlated. -Unfortunately, averaging many highly correlated quantities does not -lead to as large of a reduction in variance as averaging many -uncorrelated quantities. In particular, this means that bagging will -not lead to a substantial reduction in variance over a single tree in -this setting. +

    As another example, consider the cost function we defined for linear regression

    +$$ +C(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f(x_i))^2. +$$ + +

    In this case the function \( f(x) \) was replaced by the design matrix +\( \boldsymbol{X} \) and the unknown linear regression parameters \( \boldsymbol{\beta} \), +that is \( \boldsymbol{f}=\boldsymbol{X}\boldsymbol{\beta} \). In linear regression we can +simply invert a matrix and obtain the parameters \( \beta \) by

    +$$ +\boldsymbol{\beta}=\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}. +$$ + +

    In iterative fitting or additive modeling, we minimize the cost function with respect to the parameters \( \beta_m \) and \( \gamma_m \).

    +

    diff --git a/doc/pub/week44/html/._week44-bs060.html b/doc/pub/week44/html/._week44-bs060.html index 0f11baaba..321707369 100644 --- a/doc/pub/week44/html/._week44-bs060.html +++ b/doc/pub/week44/html/._week44-bs060.html @@ -38,10 +38,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Overview of week 44', 2, None, 'overview-of-week-44'), ('Digression First', 2, None, 'digression-first'), - ('A short Discussion of Project 2', - 2, - None, - 'a-short-discussion-of-project-2'), ('Decision trees, overarching aims', 2, None, @@ -149,10 +145,10 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'computing-the-gini-factor'), - ('Another example, the moons again', + ('Another example, the moons', 2, None, - 'another-example-the-moons-again'), + 'another-example-the-moons'), ('Playing around with regions', 2, None, @@ -173,26 +169,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -205,6 +181,13 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d None, 'using-the-voting-classifier'), ('Voting and Bagging', 2, None, 'voting-and-bagging'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Random forests', 2, None, 'random-forests'), ('Random Forest Algorithm', 2, None, 'random-forest-algorithm'), ('Random Forests Compared with other Methods on the Cancer Data', @@ -214,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -251,66 +265,71 @@ MathJax.Hub.Config({ @@ -322,23 +341,25 @@ MathJax.Hub.Config({

     

     

     

    -

    Random Forest Algorithm

    -

    The algorithm described here can be applied to both classification and regression problems.

    +

    Iterative Fitting, Regression and Squared-error Cost Function

    + +

    The way we proceed is as follows (here we specialize to the squared-error cost function)

    -

    We will grow of forest of say \( B \) trees.

      -
    1. For \( b=1:B \)
      2. -
        -
      • Draw a bootstrap sample from the training data organized in our \( \boldsymbol{X} \) matrix.
        • -
      • We grow then a random forest tree \( T_b \) based on the bootstrapped data by repeating the steps outlined till we reach the maximum node size is reached
        • -
          -
        1. we select \( m \le p \) variables at random from the \( p \) predictors/features
          2. -
        2. pick the best split point among the \( m \) features using for example the CART algorithm and create a new node
          3. -
        3. split the node into daughter nodes
          4. +
        4. Establish a cost function, here \( {\cal C}(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f_M(x_i))^2 \) with \( f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m) \).
          5. +
        5. Initialize with a guess \( f_0(x) \). It could be one or even zero or some random numbers.
          6. +
        6. For \( m=1:M \) +
            +
          1. minimize \( \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2 \) wrt \( \gamma \) and \( \beta \)
            2. +
          2. This gives the optimal values \( \beta_m \) and \( \gamma_m \)
            3. +
          3. Determine then the new values \( f_m(x)=f_{m-1}(x) +\beta_m b(x;\gamma_m) \)
          -
      -
    2. Output then the ensemble of trees \( \{T_b\}_1^{B} \) and make predictions for either a regression type of problem or a classification type of problem.
    +

    We could use any of the algorithms we have discussed till now. If we +use trees, \( \gamma \) parameterizes the split variables and split points +at the internal nodes, and the predictions at the terminal nodes. +

    +

    diff --git a/doc/pub/week44/html/._week44-bs061.html b/doc/pub/week44/html/._week44-bs061.html index 086dd43e9..bb86f53ad 100644 --- a/doc/pub/week44/html/._week44-bs061.html +++ b/doc/pub/week44/html/._week44-bs061.html @@ -38,10 +38,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Overview of week 44', 2, None, 'overview-of-week-44'), ('Digression First', 2, None, 'digression-first'), - ('A short Discussion of Project 2', - 2, - None, - 'a-short-discussion-of-project-2'), ('Decision trees, overarching aims', 2, None, @@ -149,10 +145,10 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'computing-the-gini-factor'), - ('Another example, the moons again', + ('Another example, the moons', 2, None, - 'another-example-the-moons-again'), + 'another-example-the-moons'), ('Playing around with regions', 2, None, @@ -173,26 +169,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -205,6 +181,13 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d None, 'using-the-voting-classifier'), ('Voting and Bagging', 2, None, 'voting-and-bagging'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Random forests', 2, None, 'random-forests'), ('Random Forest Algorithm', 2, None, 'random-forest-algorithm'), ('Random Forests Compared with other Methods on the Cancer Data', @@ -214,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -251,66 +265,71 @@ MathJax.Hub.Config({ @@ -322,103 +341,46 @@ MathJax.Hub.Config({

     

     

     

    -

    Random Forests Compared with other Methods on the Cancer Data

    +

    Squared-Error Example and Iterative Fitting

    - -
    -
    -
    -
    -
    - 
    import matplotlib.pyplot as plt
-import numpy as np
-from sklearn.model_selection import  train_test_split 
-from sklearn.datasets import load_breast_cancer
-from sklearn.svm import SVC
-from sklearn.linear_model import LogisticRegression
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.ensemble import BaggingClassifier
+
    To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function.
    
 
-# Load the data
-cancer = load_breast_cancer()
+
    For simplicity we assume also that our functions \( b(x;\gamma)=1+\gamma x \). 
    
 
-X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-print(X_train.shape)
-print(X_test.shape)
-# Logistic Regression
-logreg = LogisticRegression(solver='lbfgs')
-logreg.fit(X_train, y_train)
-print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
-# Support vector machine
-svm = SVC(gamma='auto', C=100)
-svm.fit(X_train, y_train)
-print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
-# Decision Trees
-deep_tree_clf = DecisionTreeClassifier(max_depth=None)
-deep_tree_clf.fit(X_train, y_train)
-print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
-#now scale the data
-from sklearn.preprocessing import StandardScaler
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-# Logistic Regression
-logreg.fit(X_train_scaled, y_train)
-print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Support Vector Machine
-svm.fit(X_train_scaled, y_train)
-print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Decision Trees
-deep_tree_clf.fit(X_train_scaled, y_train)
-print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
+
    This means that for every iteration \( m \), we need to optimize
    
 
+$$
+(\beta_m,\gamma_m) = \mathrm{argmin}_{\beta,\lambda}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2=\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta(1+\gamma x_i))^2.
+$$
 
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.preprocessing import LabelEncoder
-from sklearn.model_selection import cross_validate
-# Data set not specificied
-#Instantiate the model with 500 trees and entropy as splitting criteria
-Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy")
-Random_Forest_model.fit(X_train_scaled, y_train)
-#Cross validation
-accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score']
-print(accuracy)
-print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test)))
+
    We start our iteration by simply setting \( f_0(x)=0 \). 
+Taking the derivatives  with respect to \( \beta \) and \( \gamma \) we obtain
+
    
+$$
+\frac{\partial {\cal C}}{\partial \beta} = -2\sum_{i}(1+\gamma x_i)(y_i-\beta(1+\gamma x_i))=0,
+$$
 
+
    and
    
+$$
+\frac{\partial {\cal C}}{\partial \gamma} =-2\sum_{i}\beta x_i(y_i-\beta(1+\gamma x_i))=0.
+$$
 
-import scikitplot as skplt
-y_pred = Random_Forest_model.predict(X_test_scaled)
-skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
-plt.show()
-y_probas = Random_Forest_model.predict_proba(X_test_scaled)
-skplt.metrics.plot_roc(y_test, y_probas)
-plt.show()
-skplt.metrics.plot_cumulative_gain(y_test, y_probas)
-plt.show()
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +

    We can then rewrite these equations as (defining \( \boldsymbol{w}=\boldsymbol{e}+\gamma \boldsymbol{x}) \) with \( \boldsymbol{e} \) being the unit vector)

    +$$ +\gamma \boldsymbol{w}^T(\boldsymbol{y}-\beta\gamma \boldsymbol{w})=0, +$$ -

    Recall that the cumulative gains curve shows the percentage of the -overall number of cases in a given category gained by targeting a -percentage of the total number of cases. +

    which gives us \( \beta = \boldsymbol{w}^T\boldsymbol{y}/(\boldsymbol{w}^T\boldsymbol{w}) \). Similarly we have

    +$$ +\beta\gamma \boldsymbol{x}^T(\boldsymbol{y}-\beta(1+\gamma \boldsymbol{x}))=0, +$$ + +

    which leads to \( \gamma =(\boldsymbol{x}^T\boldsymbol{y}-\beta\boldsymbol{x}^T\boldsymbol{e})/(\beta\boldsymbol{x}^T\boldsymbol{x}) \). Inserting +for \( \beta \) gives us an equation for \( \gamma \). This is a non-linear equation in the unknown \( \gamma \) and has to be solved numerically.

    -

    Similarly, the receiver operating characteristic curve, or ROC curve, -displays the diagnostic ability of a binary classifier system as its -discrimination threshold is varied. It plots the true positive rate against the false positive rate. +

    The solution to these two equations gives us in turn \( \beta_1 \) and \( \gamma_1 \) leading to the new expression for \( f_1(x) \) as +\( f_1(x) = \beta_1(1+\gamma_1x) \). Doing this \( M \) times results in our final estimate for the function \( f \).

    @@ -437,6 +399,11 @@ discrimination threshold is varied. It plots the true positive rate against the

  • 61
  • 62
  • 63
    • +
  • 64
    • +
  • 65
    • +
  • 66
    • +
  • 67
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs062.html b/doc/pub/week44/html/._week44-bs062.html index f711b3d11..17caedb98 100644 --- a/doc/pub/week44/html/._week44-bs062.html +++ b/doc/pub/week44/html/._week44-bs062.html @@ -38,10 +38,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Overview of week 44', 2, None, 'overview-of-week-44'), ('Digression First', 2, None, 'digression-first'), - ('A short Discussion of Project 2', - 2, - None, - 'a-short-discussion-of-project-2'), ('Decision trees, overarching aims', 2, None, @@ -149,10 +145,10 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'computing-the-gini-factor'), - ('Another example, the moons again', + ('Another example, the moons', 2, None, - 'another-example-the-moons-again'), + 'another-example-the-moons'), ('Playing around with regions', 2, None, @@ -173,26 +169,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -205,6 +181,13 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d None, 'using-the-voting-classifier'), ('Voting and Bagging', 2, None, 'voting-and-bagging'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Random forests', 2, None, 'random-forests'), ('Random Forest Algorithm', 2, None, 'random-forest-algorithm'), ('Random Forests Compared with other Methods on the Cancer Data', @@ -214,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -251,66 +265,71 @@ MathJax.Hub.Config({ @@ -322,57 +341,35 @@ MathJax.Hub.Config({

     

     

     

    -

    Compare Bagging on Trees with Random Forests

    +

    Iterative Fitting, Classification and AdaBoost

    - -
    -
    -
    -
    -
    - 
    bag_clf = BaggingClassifier(
-    DecisionTreeClassifier(splitter="random", max_leaf_nodes=16, random_state=42),
-    n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1, random_state=42)
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -
    -
    -
    -
    -
    - 
    bag_clf.fit(X_train, y_train)
-y_pred = bag_clf.predict(X_test)
-from sklearn.ensemble import RandomForestClassifier
-rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)
-rnd_clf.fit(X_train, y_train)
-y_pred_rf = rnd_clf.predict(X_test)
-np.sum(y_pred == y_pred_rf) / len(y_pred) 
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +

    Let us consider a binary classification problem with two outcomes \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of +observations. We define a classification function \( G(x) \) which produces a prediction taking one or the other of the two values +\( \{-1,1\} \). +

    + +

    The error rate of the training sample is then

    + +$$ +\mathrm{\overline{err}}=\frac{1}{n} \sum_{i=0}^{n-1} I(y_i\ne G(x_i)). +$$ + +

    The iterative procedure starts with defining a weak classifier whose +error rate is barely better than random guessing. The iterative +procedure in boosting is to sequentially apply a weak +classification algorithm to repeatedly modified versions of the data +producing a sequence of weak classifiers \( G_m(x) \). +

    + +

    Here we will express our function \( f(x) \) in terms of \( G(x) \). That is

    +$$ +f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m), +$$ + +

    will be a function of

    +$$ +G_M(x) = \mathrm{sign} \sum_{i=1}^M \alpha_m G_m(x). +$$

    @@ -390,6 +387,12 @@ np.sum(y_pred =

  • 61
  • 62
  • 63
    • +
  • 64
    • +
  • 65
    • +
  • 66
    • +
  • 67
    • +
  • 68
    • +
  • »
    • diff --git a/doc/pub/week44/html/._week44-bs063.html b/doc/pub/week44/html/._week44-bs063.html index 838618e21..fad4a2a7e 100644 --- a/doc/pub/week44/html/._week44-bs063.html +++ b/doc/pub/week44/html/._week44-bs063.html @@ -38,10 +38,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Overview of week 44', 2, None, 'overview-of-week-44'), ('Digression First', 2, None, 'digression-first'), - ('A short Discussion of Project 2', - 2, - None, - 'a-short-discussion-of-project-2'), ('Decision trees, overarching aims', 2, None, @@ -149,40 +145,10 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'computing-the-gini-factor'), - ('Another example, the moons again', + ('Another example, the moons', 2, None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Example: Computing the Gini index', - 2, - None, - 'example-computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), + 'another-example-the-moons'), ('Playing around with regions', 2, None, @@ -203,26 +169,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -235,6 +181,13 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d None, 'using-the-voting-classifier'), ('Voting and Bagging', 2, None, 'voting-and-bagging'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Random forests', 2, None, 'random-forests'), ('Random Forest Algorithm', 2, None, 'random-forest-algorithm'), ('Random Forests Compared with other Methods on the Cancer Data', @@ -244,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -281,75 +265,71 @@ MathJax.Hub.Config({ @@ -361,32 +341,29 @@ MathJax.Hub.Config({

     

     

     

    -

    Tossing coins

    +

    Adaptive Boosting, AdaBoost

    -

    The simplest case is a so-called voting ensemble. To illustrate this, -think of yourself tossing coins with a biased outcome of 51 per cent -for heads and 49% for tails. With only few tosses, -you may not clearly see this distribution for heads and tails. However, after some -thousands of tosses, there will be a clear majority of heads. With 2000 tosses -you should see approximately 1020 heads and 980 tails. +

    In our iterative procedure we define thus

    +$$ +f_m(x) = f_{m-1}(x)+\beta_mG_m(x). +$$ + +

    The simplest possible cost function which leads (also simple from a computational point of view) to the AdaBoost algorithm is the +exponential cost/loss function defined as +

    +$$ +C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}\exp{(-y_i(f_{m-1}(x_i)+\beta G(x_i))}. +$$ + +

    We optimize \( \beta \) and \( G \) for each value of \( m=1:M \) as we did in the regression case. +This is normally done in two steps. Let us however first rewrite the cost function as

    -

    We can then state that the outcome is a clear majority of heads. If -you do this ten thousand times, it is easy to see that there is a 97% -likelihood of a majority of heads. -

    +$$ +C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}w_i^{m}\exp{(-y_i\beta G(x_i))}, +$$ -

    Another example would be to collect all polls before an -election. Different polls may show different likelihoods for a -candidate winning with say a majority of the popular vote. The majority vote -would then consist in many polls indicating that this candidate will -actually win. -

    - -

    The example here shows how we can implement the coin tossing case, -clealry demostrating that after some tosses we see the law of large -numbers kicking in. -

    +

    where we have defined \( w_i^m= \exp{(-y_if_{m-1}(x_i))} \).

    @@ -407,10 +384,6 @@ numbers kicking in.

  • 66
  • 67
  • 68
    • -
  • 69
    • -
  • 70
    • -
  • 71
    • -
  • 72
  • »
    • diff --git a/doc/pub/week44/html/week44-bs.html b/doc/pub/week44/html/week44-bs.html index 3e2d78be7..b05de5cdb 100644 --- a/doc/pub/week44/html/week44-bs.html +++ b/doc/pub/week44/html/week44-bs.html @@ -197,7 +197,38 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -289,6 +320,16 @@ MathJax.Hub.Config({
  • Random Forest Algorithm
  • Random Forests Compared with other Methods on the Cancer Data
  • Compare Bagging on Trees with Random Forests
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • @@ -318,7 +359,7 @@ MathJax.Hub.Config({
    -

    Nov 3, 2022

    +

    Nov 4, 2022


    @@ -343,7 +384,7 @@ MathJax.Hub.Config({
  • 9
  • 10
  • ...
    • -
  • 58
    • +
  • 68
  • »
    • diff --git a/doc/pub/week44/html/week44-reveal.html b/doc/pub/week44/html/week44-reveal.html index 08a9f3bba..68d356e55 100644 --- a/doc/pub/week44/html/week44-reveal.html +++ b/doc/pub/week44/html/week44-reveal.html @@ -184,7 +184,7 @@ MathJax.Hub.Config({
    -

    Nov 3, 2022

    +

    Nov 4, 2022


    @@ -2467,6 +2467,391 @@ np.sum(y_pred == y_pred_rf) / len(y_pred) +
    +

    Boosting, a Bird's Eye View

    + +

    The basic idea is to combine weak classifiers in order to create a good +classifier. With a weak classifier we often intend a classifier which +produces results which are only slightly better than we would get by +random guesses. +

    + +

    This is done by applying in an iterative way a weak (or a standard +classifier like decision trees) to modify the data. In each iteration +we emphasize those observations which are misclassified by weighting +them with a factor. +

    +
    + +
    +

    What is boosting? Additive Modelling/Iterative Fitting

    + +

    Boosting is a way of fitting an additive expansion in a set of +elementary basis functions like for example some simple polynomials. +Assume for example that we have a function +

    +

     
    +$$ +f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m), +$$ +

     
    + +

    where \( \beta_m \) are the expansion parameters to be determined in a +minimization process and \( b(x;\gamma_m) \) are some simple functions of +the multivariable parameter \( x \) which is characterized by the +parameters \( \gamma_m \). +

    + +

    As an example, consider the Sigmoid function we used in logistic +regression. In that case, we can translate the function +\( b(x;\gamma_m) \) into the Sigmoid function +

    + +

     
    +$$ +\sigma(t) = \frac{1}{1+\exp{(-t)}}, +$$ +

     
    + +

    where \( t=\gamma_0+\gamma_1 x \) and the parameters \( \gamma_0 \) and +\( \gamma_1 \) were determined by the Logistic Regression fitting +algorithm. +

    + +

    As another example, consider the cost function we defined for linear regression

    +

     
    +$$ +C(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f(x_i))^2. +$$ +

     
    + +

    In this case the function \( f(x) \) was replaced by the design matrix +\( \boldsymbol{X} \) and the unknown linear regression parameters \( \boldsymbol{\beta} \), +that is \( \boldsymbol{f}=\boldsymbol{X}\boldsymbol{\beta} \). In linear regression we can +simply invert a matrix and obtain the parameters \( \beta \) by +

    + +

     
    +$$ +\boldsymbol{\beta}=\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}. +$$ +

     
    + +

    In iterative fitting or additive modeling, we minimize the cost function with respect to the parameters \( \beta_m \) and \( \gamma_m \).

    +
    + +
    +

    Iterative Fitting, Regression and Squared-error Cost Function

    + +

    The way we proceed is as follows (here we specialize to the squared-error cost function)

    + +
      +

    1. Establish a cost function, here \( {\cal C}(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f_M(x_i))^2 \) with \( f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m) \).
      2. +

    2. Initialize with a guess \( f_0(x) \). It could be one or even zero or some random numbers.
      3. +

    3. For \( m=1:M \) +
        +

      1. minimize \( \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2 \) wrt \( \gamma \) and \( \beta \)
        2. +

      2. This gives the optimal values \( \beta_m \) and \( \gamma_m \)
        3. +

      3. Determine then the new values \( f_m(x)=f_{m-1}(x) +\beta_m b(x;\gamma_m) \)
        4. +
      +

      +

    +

    +

    We could use any of the algorithms we have discussed till now. If we +use trees, \( \gamma \) parameterizes the split variables and split points +at the internal nodes, and the predictions at the terminal nodes. +

    +
    + +
    +

    Squared-Error Example and Iterative Fitting

    + +

    To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function.

    + +

    For simplicity we assume also that our functions \( b(x;\gamma)=1+\gamma x \).

    + +

    This means that for every iteration \( m \), we need to optimize

    + +

     
    +$$ +(\beta_m,\gamma_m) = \mathrm{argmin}_{\beta,\lambda}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2=\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta(1+\gamma x_i))^2. +$$ +

     
    + +

    We start our iteration by simply setting \( f_0(x)=0 \). +Taking the derivatives with respect to \( \beta \) and \( \gamma \) we obtain +

    +

     
    +$$ +\frac{\partial {\cal C}}{\partial \beta} = -2\sum_{i}(1+\gamma x_i)(y_i-\beta(1+\gamma x_i))=0, +$$ +

     
    + +

    and

    +

     
    +$$ +\frac{\partial {\cal C}}{\partial \gamma} =-2\sum_{i}\beta x_i(y_i-\beta(1+\gamma x_i))=0. +$$ +

     
    + +

    We can then rewrite these equations as (defining \( \boldsymbol{w}=\boldsymbol{e}+\gamma \boldsymbol{x}) \) with \( \boldsymbol{e} \) being the unit vector)

    +

     
    +$$ +\gamma \boldsymbol{w}^T(\boldsymbol{y}-\beta\gamma \boldsymbol{w})=0, +$$ +

     
    + +

    which gives us \( \beta = \boldsymbol{w}^T\boldsymbol{y}/(\boldsymbol{w}^T\boldsymbol{w}) \). Similarly we have

    +

     
    +$$ +\beta\gamma \boldsymbol{x}^T(\boldsymbol{y}-\beta(1+\gamma \boldsymbol{x}))=0, +$$ +

     
    + +

    which leads to \( \gamma =(\boldsymbol{x}^T\boldsymbol{y}-\beta\boldsymbol{x}^T\boldsymbol{e})/(\beta\boldsymbol{x}^T\boldsymbol{x}) \). Inserting +for \( \beta \) gives us an equation for \( \gamma \). This is a non-linear equation in the unknown \( \gamma \) and has to be solved numerically. +

    + +

    The solution to these two equations gives us in turn \( \beta_1 \) and \( \gamma_1 \) leading to the new expression for \( f_1(x) \) as +\( f_1(x) = \beta_1(1+\gamma_1x) \). Doing this \( M \) times results in our final estimate for the function \( f \). +

    +
    + +
    +

    Iterative Fitting, Classification and AdaBoost

    + +

    Let us consider a binary classification problem with two outcomes \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of +observations. We define a classification function \( G(x) \) which produces a prediction taking one or the other of the two values +\( \{-1,1\} \). +

    + +

    The error rate of the training sample is then

    + +

     
    +$$ +\mathrm{\overline{err}}=\frac{1}{n} \sum_{i=0}^{n-1} I(y_i\ne G(x_i)). +$$ +

     
    + +

    The iterative procedure starts with defining a weak classifier whose +error rate is barely better than random guessing. The iterative +procedure in boosting is to sequentially apply a weak +classification algorithm to repeatedly modified versions of the data +producing a sequence of weak classifiers \( G_m(x) \). +

    + +

    Here we will express our function \( f(x) \) in terms of \( G(x) \). That is

    +

     
    +$$ +f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m), +$$ +

     
    + +

    will be a function of

    +

     
    +$$ +G_M(x) = \mathrm{sign} \sum_{i=1}^M \alpha_m G_m(x). +$$ +

     
    +

    + +
    +

    Adaptive Boosting, AdaBoost

    + +

    In our iterative procedure we define thus

    +

     
    +$$ +f_m(x) = f_{m-1}(x)+\beta_mG_m(x). +$$ +

     
    + +

    The simplest possible cost function which leads (also simple from a computational point of view) to the AdaBoost algorithm is the +exponential cost/loss function defined as +

    +

     
    +$$ +C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}\exp{(-y_i(f_{m-1}(x_i)+\beta G(x_i))}. +$$ +

     
    + +

    We optimize \( \beta \) and \( G \) for each value of \( m=1:M \) as we did in the regression case. +This is normally done in two steps. Let us however first rewrite the cost function as +

    + +

     
    +$$ +C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}w_i^{m}\exp{(-y_i\beta G(x_i))}, +$$ +

     
    + +

    where we have defined \( w_i^m= \exp{(-y_if_{m-1}(x_i))} \).

    +
    + +
    +

    Building up AdaBoost

    + +

    First, for any \( \beta > 0 \), we optimize \( G \) by setting

    +

     
    +$$ +G_m(x) = \mathrm{sign} \sum_{i=0}^{n-1} w_i^m I(y_i \ne G_(x_i)), +$$ +

     
    + +

    which is the classifier that minimizes the weighted error rate in predicting \( y \).

    + +

    We can do this by rewriting

    +

     
    +$$ +\exp{-(\beta)}\sum_{y_i=G(x_i)}w_i^m+\exp{(\beta)}\sum_{y_i\ne G(x_i)}w_i^m, +$$ +

     
    + +

    which can be rewritten as

    +

     
    +$$ +(\exp{(\beta)}-\exp{-(\beta)})\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i))+\exp{(-\beta)}\sum_{i=0}^{n-1}w_i^m=0, +$$ +

     
    + +

    which leads to

    +

     
    +$$ +\beta_m = \frac{1}{2}\log{\frac{1-\mathrm{\overline{err}}}{\mathrm{\overline{err}}}}, +$$ +

     
    + +

    where we have redefined the error as

    +

     
    +$$ +\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i)}{\sum_{i=0}^{n-1}w_i^m}, +$$ +

     
    + +

    which leads to an update of

    +

     
    +$$ +f_m(x) = f_{m-1}(x) +\beta_m G_m(x). +$$ +

     
    + +

    This leads to the new weights

    +

     
    +$$ +w_i^{m+1} = w_i^m \exp{(-y_i\beta_m G_m(x_i))} +$$ +

     
    +

    + +
    +

    Adaptive boosting: AdaBoost, Basic Algorithm

    + +

    The algorithm here is rather straightforward. Assume that our weak +classifier is a decision tree and we consider a binary set of outputs +with \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of +observations. Our design matrix is given in terms of the +feature/predictor vectors +\( \boldsymbol{X}=[\boldsymbol{x}_0\boldsymbol{x}_1\dots\boldsymbol{x}_{p-1}] \). Finally, we define also a +classifier determined by our data via a function \( G(x) \). This function tells us how well we are able to classify our outputs/targets \( \boldsymbol{y} \). +

    + +

    We have already defined the misclassification error \( \mathrm{err} \) as

    +

     
    +$$ +\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(x_i)), +$$ +

     
    + +

    where the function \( I() \) is one if we misclassify and zero if we classify correctly.

    +
    + +
    +

    Basic Steps of AdaBoost

    + +

    With the above definitions we are now ready to set up the algorithm for AdaBoost. +The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases. +

    +
      +

    1. We start by initializing all weights to \( w_i = 1/n \), with \( i=0,1,2,\dots n-1 \). It is easy to see that we must have \( \sum_{i=0}^{n-1}w_i = 1 \).
      2. +

    2. We rewrite the misclassification error as
      3. +
    +

    +

     
    +$$ +\mathrm{\overline{err}}_m=\frac{\sum_{i=0}^{n-1}w_i^m I(y_i\ne G(x_i))}{\sum_{i=0}^{n-1}w_i}, +$$ +

     
    + +

      +

    1. Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree. +
        +

      1. Fit then a given classifier to the training set using the weights \( w_i \).
        2. +

      2. Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.
        3. +

      3. Define a quantity \( \alpha_{m} = \log{(1-\mathrm{\overline{err}}_m)/\mathrm{\overline{err}}_m} \)
        4. +

      4. Set the new weights to \( w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(x_i)} \).
        5. +
      +

      +

    2. Compute the new classifier \( G(x)= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(x_i) \).
      3. +
    +

    +

    For the iterations with \( m \le 2 \) the weights are modified +individually at each steps. The observations which were misclassified +at iteration \( m-1 \) have a weight which is larger than those which were +classified properly. As this proceeds, the observations which were +difficult to classifiy correctly are given a larger influence. Each +new classification step \( m \) is then forced to concentrate on those +observations that are missed in the previous iterations. +

    +
    + +
    +

    AdaBoost Examples

    + +

    Using Scikit-Learn it is easy to apply the adaptive boosting algorithm, as done here.

    + + + +
    +
    +
    +
    +
    + 
    from sklearn.ensemble import AdaBoostClassifier
+
+ada_clf = AdaBoostClassifier(
+    DecisionTreeClassifier(max_depth=1), n_estimators=200,
+    algorithm="SAMME.R", learning_rate=0.5, random_state=42)
+ada_clf.fit(X_train, y_train)
+
+from sklearn.ensemble import AdaBoostClassifier
+
+ada_clf = AdaBoostClassifier(
+    DecisionTreeClassifier(max_depth=1), n_estimators=200,
+    algorithm="SAMME.R", learning_rate=0.5, random_state=42)
+ada_clf.fit(X_train_scaled, y_train)
+y_pred = ada_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = ada_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + diff --git a/doc/pub/week44/html/week44-solarized.html b/doc/pub/week44/html/week44-solarized.html index 823252880..8a3b2a1fd 100644 --- a/doc/pub/week44/html/week44-solarized.html +++ b/doc/pub/week44/html/week44-solarized.html @@ -224,7 +224,38 @@ div.toc p,a { ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -262,7 +293,7 @@ MathJax.Hub.Config({
    -

    Nov 3, 2022

    +

    Nov 4, 2022


    @@ -2449,6 +2480,330 @@ np.sum(y_pred == y_pred_rf) / len(y_pred) +









    +

    Boosting, a Bird's Eye View

    + +

    The basic idea is to combine weak classifiers in order to create a good +classifier. With a weak classifier we often intend a classifier which +produces results which are only slightly better than we would get by +random guesses. +

    + +

    This is done by applying in an iterative way a weak (or a standard +classifier like decision trees) to modify the data. In each iteration +we emphasize those observations which are misclassified by weighting +them with a factor. +

    + +









    +

    What is boosting? Additive Modelling/Iterative Fitting

    + +

    Boosting is a way of fitting an additive expansion in a set of +elementary basis functions like for example some simple polynomials. +Assume for example that we have a function +

    +$$ +f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m), +$$ + +

    where \( \beta_m \) are the expansion parameters to be determined in a +minimization process and \( b(x;\gamma_m) \) are some simple functions of +the multivariable parameter \( x \) which is characterized by the +parameters \( \gamma_m \). +

    + +

    As an example, consider the Sigmoid function we used in logistic +regression. In that case, we can translate the function +\( b(x;\gamma_m) \) into the Sigmoid function +

    + +$$ +\sigma(t) = \frac{1}{1+\exp{(-t)}}, +$$ + +

    where \( t=\gamma_0+\gamma_1 x \) and the parameters \( \gamma_0 \) and +\( \gamma_1 \) were determined by the Logistic Regression fitting +algorithm. +

    + +

    As another example, consider the cost function we defined for linear regression

    +$$ +C(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f(x_i))^2. +$$ + +

    In this case the function \( f(x) \) was replaced by the design matrix +\( \boldsymbol{X} \) and the unknown linear regression parameters \( \boldsymbol{\beta} \), +that is \( \boldsymbol{f}=\boldsymbol{X}\boldsymbol{\beta} \). In linear regression we can +simply invert a matrix and obtain the parameters \( \beta \) by +

    + +$$ +\boldsymbol{\beta}=\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}. +$$ + +

    In iterative fitting or additive modeling, we minimize the cost function with respect to the parameters \( \beta_m \) and \( \gamma_m \).

    + +









    +

    Iterative Fitting, Regression and Squared-error Cost Function

    + +

    The way we proceed is as follows (here we specialize to the squared-error cost function)

    + +
      +
    1. Establish a cost function, here \( {\cal C}(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f_M(x_i))^2 \) with \( f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m) \).
      2. +
    2. Initialize with a guess \( f_0(x) \). It could be one or even zero or some random numbers.
      3. +
    3. For \( m=1:M \) +
        +
      1. minimize \( \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2 \) wrt \( \gamma \) and \( \beta \)
        2. +
      2. This gives the optimal values \( \beta_m \) and \( \gamma_m \)
        3. +
      3. Determine then the new values \( f_m(x)=f_{m-1}(x) +\beta_m b(x;\gamma_m) \)
        4. +
      +
    +

    We could use any of the algorithms we have discussed till now. If we +use trees, \( \gamma \) parameterizes the split variables and split points +at the internal nodes, and the predictions at the terminal nodes. +

    + +









    +

    Squared-Error Example and Iterative Fitting

    + +

    To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function.

    + +

    For simplicity we assume also that our functions \( b(x;\gamma)=1+\gamma x \).

    + +

    This means that for every iteration \( m \), we need to optimize

    + +$$ +(\beta_m,\gamma_m) = \mathrm{argmin}_{\beta,\lambda}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2=\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta(1+\gamma x_i))^2. +$$ + +

    We start our iteration by simply setting \( f_0(x)=0 \). +Taking the derivatives with respect to \( \beta \) and \( \gamma \) we obtain +

    +$$ +\frac{\partial {\cal C}}{\partial \beta} = -2\sum_{i}(1+\gamma x_i)(y_i-\beta(1+\gamma x_i))=0, +$$ + +

    and

    +$$ +\frac{\partial {\cal C}}{\partial \gamma} =-2\sum_{i}\beta x_i(y_i-\beta(1+\gamma x_i))=0. +$$ + +

    We can then rewrite these equations as (defining \( \boldsymbol{w}=\boldsymbol{e}+\gamma \boldsymbol{x}) \) with \( \boldsymbol{e} \) being the unit vector)

    +$$ +\gamma \boldsymbol{w}^T(\boldsymbol{y}-\beta\gamma \boldsymbol{w})=0, +$$ + +

    which gives us \( \beta = \boldsymbol{w}^T\boldsymbol{y}/(\boldsymbol{w}^T\boldsymbol{w}) \). Similarly we have

    +$$ +\beta\gamma \boldsymbol{x}^T(\boldsymbol{y}-\beta(1+\gamma \boldsymbol{x}))=0, +$$ + +

    which leads to \( \gamma =(\boldsymbol{x}^T\boldsymbol{y}-\beta\boldsymbol{x}^T\boldsymbol{e})/(\beta\boldsymbol{x}^T\boldsymbol{x}) \). Inserting +for \( \beta \) gives us an equation for \( \gamma \). This is a non-linear equation in the unknown \( \gamma \) and has to be solved numerically. +

    + +

    The solution to these two equations gives us in turn \( \beta_1 \) and \( \gamma_1 \) leading to the new expression for \( f_1(x) \) as +\( f_1(x) = \beta_1(1+\gamma_1x) \). Doing this \( M \) times results in our final estimate for the function \( f \). +

    + +









    +

    Iterative Fitting, Classification and AdaBoost

    + +

    Let us consider a binary classification problem with two outcomes \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of +observations. We define a classification function \( G(x) \) which produces a prediction taking one or the other of the two values +\( \{-1,1\} \). +

    + +

    The error rate of the training sample is then

    + +$$ +\mathrm{\overline{err}}=\frac{1}{n} \sum_{i=0}^{n-1} I(y_i\ne G(x_i)). +$$ + +

    The iterative procedure starts with defining a weak classifier whose +error rate is barely better than random guessing. The iterative +procedure in boosting is to sequentially apply a weak +classification algorithm to repeatedly modified versions of the data +producing a sequence of weak classifiers \( G_m(x) \). +

    + +

    Here we will express our function \( f(x) \) in terms of \( G(x) \). That is

    +$$ +f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m), +$$ + +

    will be a function of

    +$$ +G_M(x) = \mathrm{sign} \sum_{i=1}^M \alpha_m G_m(x). +$$ + + +









    +

    Adaptive Boosting, AdaBoost

    + +

    In our iterative procedure we define thus

    +$$ +f_m(x) = f_{m-1}(x)+\beta_mG_m(x). +$$ + +

    The simplest possible cost function which leads (also simple from a computational point of view) to the AdaBoost algorithm is the +exponential cost/loss function defined as +

    +$$ +C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}\exp{(-y_i(f_{m-1}(x_i)+\beta G(x_i))}. +$$ + +

    We optimize \( \beta \) and \( G \) for each value of \( m=1:M \) as we did in the regression case. +This is normally done in two steps. Let us however first rewrite the cost function as +

    + +$$ +C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}w_i^{m}\exp{(-y_i\beta G(x_i))}, +$$ + +

    where we have defined \( w_i^m= \exp{(-y_if_{m-1}(x_i))} \).

    + +









    +

    Building up AdaBoost

    + +

    First, for any \( \beta > 0 \), we optimize \( G \) by setting

    +$$ +G_m(x) = \mathrm{sign} \sum_{i=0}^{n-1} w_i^m I(y_i \ne G_(x_i)), +$$ + +

    which is the classifier that minimizes the weighted error rate in predicting \( y \).

    + +

    We can do this by rewriting

    +$$ +\exp{-(\beta)}\sum_{y_i=G(x_i)}w_i^m+\exp{(\beta)}\sum_{y_i\ne G(x_i)}w_i^m, +$$ + +

    which can be rewritten as

    +$$ +(\exp{(\beta)}-\exp{-(\beta)})\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i))+\exp{(-\beta)}\sum_{i=0}^{n-1}w_i^m=0, +$$ + +

    which leads to

    +$$ +\beta_m = \frac{1}{2}\log{\frac{1-\mathrm{\overline{err}}}{\mathrm{\overline{err}}}}, +$$ + +

    where we have redefined the error as

    +$$ +\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i)}{\sum_{i=0}^{n-1}w_i^m}, +$$ + +

    which leads to an update of

    +$$ +f_m(x) = f_{m-1}(x) +\beta_m G_m(x). +$$ + +

    This leads to the new weights

    +$$ +w_i^{m+1} = w_i^m \exp{(-y_i\beta_m G_m(x_i))} +$$ + + +









    +

    Adaptive boosting: AdaBoost, Basic Algorithm

    + +

    The algorithm here is rather straightforward. Assume that our weak +classifier is a decision tree and we consider a binary set of outputs +with \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of +observations. Our design matrix is given in terms of the +feature/predictor vectors +\( \boldsymbol{X}=[\boldsymbol{x}_0\boldsymbol{x}_1\dots\boldsymbol{x}_{p-1}] \). Finally, we define also a +classifier determined by our data via a function \( G(x) \). This function tells us how well we are able to classify our outputs/targets \( \boldsymbol{y} \). +

    + +

    We have already defined the misclassification error \( \mathrm{err} \) as

    +$$ +\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(x_i)), +$$ + +

    where the function \( I() \) is one if we misclassify and zero if we classify correctly.

    + +









    +

    Basic Steps of AdaBoost

    + +

    With the above definitions we are now ready to set up the algorithm for AdaBoost. +The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases. +

    +
      +
    1. We start by initializing all weights to \( w_i = 1/n \), with \( i=0,1,2,\dots n-1 \). It is easy to see that we must have \( \sum_{i=0}^{n-1}w_i = 1 \).
      2. +
    2. We rewrite the misclassification error as
      3. +
    +$$ +\mathrm{\overline{err}}_m=\frac{\sum_{i=0}^{n-1}w_i^m I(y_i\ne G(x_i))}{\sum_{i=0}^{n-1}w_i}, +$$ + +
      +
    1. Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree. +
        +
      1. Fit then a given classifier to the training set using the weights \( w_i \).
        2. +
      2. Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.
        3. +
      3. Define a quantity \( \alpha_{m} = \log{(1-\mathrm{\overline{err}}_m)/\mathrm{\overline{err}}_m} \)
        4. +
      4. Set the new weights to \( w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(x_i)} \).
        5. +
      +
    2. Compute the new classifier \( G(x)= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(x_i) \).
      3. +
    +

    For the iterations with \( m \le 2 \) the weights are modified +individually at each steps. The observations which were misclassified +at iteration \( m-1 \) have a weight which is larger than those which were +classified properly. As this proceeds, the observations which were +difficult to classifiy correctly are given a larger influence. Each +new classification step \( m \) is then forced to concentrate on those +observations that are missed in the previous iterations. +

    + +









    +

    AdaBoost Examples

    + +

    Using Scikit-Learn it is easy to apply the adaptive boosting algorithm, as done here.

    + + + +
    +
    +
    +
    +
    + 
    from sklearn.ensemble import AdaBoostClassifier
+
+ada_clf = AdaBoostClassifier(
+    DecisionTreeClassifier(max_depth=1), n_estimators=200,
+    algorithm="SAMME.R", learning_rate=0.5, random_state=42)
+ada_clf.fit(X_train, y_train)
+
+from sklearn.ensemble import AdaBoostClassifier
+
+ada_clf = AdaBoostClassifier(
+    DecisionTreeClassifier(max_depth=1), n_estimators=200,
+    algorithm="SAMME.R", learning_rate=0.5, random_state=42)
+ada_clf.fit(X_train_scaled, y_train)
+y_pred = ada_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = ada_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    © 1999-2022, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license diff --git a/doc/pub/week44/html/week44.html b/doc/pub/week44/html/week44.html index c06462399..52ec79400 100644 --- a/doc/pub/week44/html/week44.html +++ b/doc/pub/week44/html/week44.html @@ -301,7 +301,38 @@ div.toc p,a { ('Compare Bagging on Trees with Random Forests', 2, None, - 'compare-bagging-on-trees-with-random-forests')]} + 'compare-bagging-on-trees-with-random-forests'), + ("Boosting, a Bird's Eye View", + 2, + None, + 'boosting-a-bird-s-eye-view'), + ('What is boosting? Additive Modelling/Iterative Fitting', + 2, + None, + 'what-is-boosting-additive-modelling-iterative-fitting'), + ('Iterative Fitting, Regression and Squared-error Cost Function', + 2, + None, + 'iterative-fitting-regression-and-squared-error-cost-function'), + ('Squared-Error Example and Iterative Fitting', + 2, + None, + 'squared-error-example-and-iterative-fitting'), + ('Iterative Fitting, Classification and AdaBoost', + 2, + None, + 'iterative-fitting-classification-and-adaboost'), + ('Adaptive Boosting, AdaBoost', + 2, + None, + 'adaptive-boosting-adaboost'), + ('Building up AdaBoost', 2, None, 'building-up-adaboost'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + 'adaptive-boosting-adaboost-basic-algorithm'), + ('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'), + ('AdaBoost Examples', 2, None, 'adaboost-examples')]} end of tocinfo --> @@ -339,7 +370,7 @@ MathJax.Hub.Config({

    -

    Nov 3, 2022

    +

    Nov 4, 2022


    @@ -2526,6 +2557,330 @@ np.sum(y_pred = +









    +

    Boosting, a Bird's Eye View

    + +

    The basic idea is to combine weak classifiers in order to create a good +classifier. With a weak classifier we often intend a classifier which +produces results which are only slightly better than we would get by +random guesses. +

    + +

    This is done by applying in an iterative way a weak (or a standard +classifier like decision trees) to modify the data. In each iteration +we emphasize those observations which are misclassified by weighting +them with a factor. +

    + +









    +

    What is boosting? Additive Modelling/Iterative Fitting

    + +

    Boosting is a way of fitting an additive expansion in a set of +elementary basis functions like for example some simple polynomials. +Assume for example that we have a function +

    +$$ +f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m), +$$ + +

    where \( \beta_m \) are the expansion parameters to be determined in a +minimization process and \( b(x;\gamma_m) \) are some simple functions of +the multivariable parameter \( x \) which is characterized by the +parameters \( \gamma_m \). +

    + +

    As an example, consider the Sigmoid function we used in logistic +regression. In that case, we can translate the function +\( b(x;\gamma_m) \) into the Sigmoid function +

    + +$$ +\sigma(t) = \frac{1}{1+\exp{(-t)}}, +$$ + +

    where \( t=\gamma_0+\gamma_1 x \) and the parameters \( \gamma_0 \) and +\( \gamma_1 \) were determined by the Logistic Regression fitting +algorithm. +

    + +

    As another example, consider the cost function we defined for linear regression

    +$$ +C(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f(x_i))^2. +$$ + +

    In this case the function \( f(x) \) was replaced by the design matrix +\( \boldsymbol{X} \) and the unknown linear regression parameters \( \boldsymbol{\beta} \), +that is \( \boldsymbol{f}=\boldsymbol{X}\boldsymbol{\beta} \). In linear regression we can +simply invert a matrix and obtain the parameters \( \beta \) by +

    + +$$ +\boldsymbol{\beta}=\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}. +$$ + +

    In iterative fitting or additive modeling, we minimize the cost function with respect to the parameters \( \beta_m \) and \( \gamma_m \).

    + +









    +

    Iterative Fitting, Regression and Squared-error Cost Function

    + +

    The way we proceed is as follows (here we specialize to the squared-error cost function)

    + +
      +
    1. Establish a cost function, here \( {\cal C}(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f_M(x_i))^2 \) with \( f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m) \).
      2. +
    2. Initialize with a guess \( f_0(x) \). It could be one or even zero or some random numbers.
      3. +
    3. For \( m=1:M \) +
        +
      1. minimize \( \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2 \) wrt \( \gamma \) and \( \beta \)
        2. +
      2. This gives the optimal values \( \beta_m \) and \( \gamma_m \)
        3. +
      3. Determine then the new values \( f_m(x)=f_{m-1}(x) +\beta_m b(x;\gamma_m) \)
        4. +
      +
    +

    We could use any of the algorithms we have discussed till now. If we +use trees, \( \gamma \) parameterizes the split variables and split points +at the internal nodes, and the predictions at the terminal nodes. +

    + +









    +

    Squared-Error Example and Iterative Fitting

    + +

    To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function.

    + +

    For simplicity we assume also that our functions \( b(x;\gamma)=1+\gamma x \).

    + +

    This means that for every iteration \( m \), we need to optimize

    + +$$ +(\beta_m,\gamma_m) = \mathrm{argmin}_{\beta,\lambda}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2=\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta(1+\gamma x_i))^2. +$$ + +

    We start our iteration by simply setting \( f_0(x)=0 \). +Taking the derivatives with respect to \( \beta \) and \( \gamma \) we obtain +

    +$$ +\frac{\partial {\cal C}}{\partial \beta} = -2\sum_{i}(1+\gamma x_i)(y_i-\beta(1+\gamma x_i))=0, +$$ + +

    and

    +$$ +\frac{\partial {\cal C}}{\partial \gamma} =-2\sum_{i}\beta x_i(y_i-\beta(1+\gamma x_i))=0. +$$ + +

    We can then rewrite these equations as (defining \( \boldsymbol{w}=\boldsymbol{e}+\gamma \boldsymbol{x}) \) with \( \boldsymbol{e} \) being the unit vector)

    +$$ +\gamma \boldsymbol{w}^T(\boldsymbol{y}-\beta\gamma \boldsymbol{w})=0, +$$ + +

    which gives us \( \beta = \boldsymbol{w}^T\boldsymbol{y}/(\boldsymbol{w}^T\boldsymbol{w}) \). Similarly we have

    +$$ +\beta\gamma \boldsymbol{x}^T(\boldsymbol{y}-\beta(1+\gamma \boldsymbol{x}))=0, +$$ + +

    which leads to \( \gamma =(\boldsymbol{x}^T\boldsymbol{y}-\beta\boldsymbol{x}^T\boldsymbol{e})/(\beta\boldsymbol{x}^T\boldsymbol{x}) \). Inserting +for \( \beta \) gives us an equation for \( \gamma \). This is a non-linear equation in the unknown \( \gamma \) and has to be solved numerically. +

    + +

    The solution to these two equations gives us in turn \( \beta_1 \) and \( \gamma_1 \) leading to the new expression for \( f_1(x) \) as +\( f_1(x) = \beta_1(1+\gamma_1x) \). Doing this \( M \) times results in our final estimate for the function \( f \). +

    + +









    +

    Iterative Fitting, Classification and AdaBoost

    + +

    Let us consider a binary classification problem with two outcomes \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of +observations. We define a classification function \( G(x) \) which produces a prediction taking one or the other of the two values +\( \{-1,1\} \). +

    + +

    The error rate of the training sample is then

    + +$$ +\mathrm{\overline{err}}=\frac{1}{n} \sum_{i=0}^{n-1} I(y_i\ne G(x_i)). +$$ + +

    The iterative procedure starts with defining a weak classifier whose +error rate is barely better than random guessing. The iterative +procedure in boosting is to sequentially apply a weak +classification algorithm to repeatedly modified versions of the data +producing a sequence of weak classifiers \( G_m(x) \). +

    + +

    Here we will express our function \( f(x) \) in terms of \( G(x) \). That is

    +$$ +f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m), +$$ + +

    will be a function of

    +$$ +G_M(x) = \mathrm{sign} \sum_{i=1}^M \alpha_m G_m(x). +$$ + + +









    +

    Adaptive Boosting, AdaBoost

    + +

    In our iterative procedure we define thus

    +$$ +f_m(x) = f_{m-1}(x)+\beta_mG_m(x). +$$ + +

    The simplest possible cost function which leads (also simple from a computational point of view) to the AdaBoost algorithm is the +exponential cost/loss function defined as +

    +$$ +C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}\exp{(-y_i(f_{m-1}(x_i)+\beta G(x_i))}. +$$ + +

    We optimize \( \beta \) and \( G \) for each value of \( m=1:M \) as we did in the regression case. +This is normally done in two steps. Let us however first rewrite the cost function as +

    + +$$ +C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}w_i^{m}\exp{(-y_i\beta G(x_i))}, +$$ + +

    where we have defined \( w_i^m= \exp{(-y_if_{m-1}(x_i))} \).

    + +









    +

    Building up AdaBoost

    + +

    First, for any \( \beta > 0 \), we optimize \( G \) by setting

    +$$ +G_m(x) = \mathrm{sign} \sum_{i=0}^{n-1} w_i^m I(y_i \ne G_(x_i)), +$$ + +

    which is the classifier that minimizes the weighted error rate in predicting \( y \).

    + +

    We can do this by rewriting

    +$$ +\exp{-(\beta)}\sum_{y_i=G(x_i)}w_i^m+\exp{(\beta)}\sum_{y_i\ne G(x_i)}w_i^m, +$$ + +

    which can be rewritten as

    +$$ +(\exp{(\beta)}-\exp{-(\beta)})\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i))+\exp{(-\beta)}\sum_{i=0}^{n-1}w_i^m=0, +$$ + +

    which leads to

    +$$ +\beta_m = \frac{1}{2}\log{\frac{1-\mathrm{\overline{err}}}{\mathrm{\overline{err}}}}, +$$ + +

    where we have redefined the error as

    +$$ +\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i)}{\sum_{i=0}^{n-1}w_i^m}, +$$ + +

    which leads to an update of

    +$$ +f_m(x) = f_{m-1}(x) +\beta_m G_m(x). +$$ + +

    This leads to the new weights

    +$$ +w_i^{m+1} = w_i^m \exp{(-y_i\beta_m G_m(x_i))} +$$ + + +









    +

    Adaptive boosting: AdaBoost, Basic Algorithm

    + +

    The algorithm here is rather straightforward. Assume that our weak +classifier is a decision tree and we consider a binary set of outputs +with \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of +observations. Our design matrix is given in terms of the +feature/predictor vectors +\( \boldsymbol{X}=[\boldsymbol{x}_0\boldsymbol{x}_1\dots\boldsymbol{x}_{p-1}] \). Finally, we define also a +classifier determined by our data via a function \( G(x) \). This function tells us how well we are able to classify our outputs/targets \( \boldsymbol{y} \). +

    + +

    We have already defined the misclassification error \( \mathrm{err} \) as

    +$$ +\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(x_i)), +$$ + +

    where the function \( I() \) is one if we misclassify and zero if we classify correctly.

    + +









    +

    Basic Steps of AdaBoost

    + +

    With the above definitions we are now ready to set up the algorithm for AdaBoost. +The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases. +

    +
      +
    1. We start by initializing all weights to \( w_i = 1/n \), with \( i=0,1,2,\dots n-1 \). It is easy to see that we must have \( \sum_{i=0}^{n-1}w_i = 1 \).
      2. +
    2. We rewrite the misclassification error as
      3. +
    +$$ +\mathrm{\overline{err}}_m=\frac{\sum_{i=0}^{n-1}w_i^m I(y_i\ne G(x_i))}{\sum_{i=0}^{n-1}w_i}, +$$ + +
      +
    1. Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree. +
        +
      1. Fit then a given classifier to the training set using the weights \( w_i \).
        2. +
      2. Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.
        3. +
      3. Define a quantity \( \alpha_{m} = \log{(1-\mathrm{\overline{err}}_m)/\mathrm{\overline{err}}_m} \)
        4. +
      4. Set the new weights to \( w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(x_i)} \).
        5. +
      +
    2. Compute the new classifier \( G(x)= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(x_i) \).
      3. +
    +

    For the iterations with \( m \le 2 \) the weights are modified +individually at each steps. The observations which were misclassified +at iteration \( m-1 \) have a weight which is larger than those which were +classified properly. As this proceeds, the observations which were +difficult to classifiy correctly are given a larger influence. Each +new classification step \( m \) is then forced to concentrate on those +observations that are missed in the previous iterations. +

    + +









    +

    AdaBoost Examples

    + +

    Using Scikit-Learn it is easy to apply the adaptive boosting algorithm, as done here.

    + + + +
    +
    +
    +
    +
    + 
    from sklearn.ensemble import AdaBoostClassifier
+
+ada_clf = AdaBoostClassifier(
+    DecisionTreeClassifier(max_depth=1), n_estimators=200,
+    algorithm="SAMME.R", learning_rate=0.5, random_state=42)
+ada_clf.fit(X_train, y_train)
+
+from sklearn.ensemble import AdaBoostClassifier
+
+ada_clf = AdaBoostClassifier(
+    DecisionTreeClassifier(max_depth=1), n_estimators=200,
+    algorithm="SAMME.R", learning_rate=0.5, random_state=42)
+ada_clf.fit(X_train_scaled, y_train)
+y_pred = ada_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = ada_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    © 1999-2022, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license diff --git a/doc/pub/week44/ipynb/ipynb-week44-src.tar.gz b/doc/pub/week44/ipynb/ipynb-week44-src.tar.gz index 0d27bf39196ee2eae0e38b2df97c1db17d537284..1ec4fff47dd32648ccaef72eba7a80f1215f0f8f 100644 GIT binary patch delta 30 lcmeDFE!h2AkX^o;gF$g~N+WwKJ7X(5Q!6|3R(6(_S^%CA2@?PS delta 30 lcmeDFE!h2AkX^o;gTd@TawB^yJ7X(5Q!6|3R(6(_S^%JV2{Zrz diff --git a/doc/pub/week44/ipynb/week44.ipynb b/doc/pub/week44/ipynb/week44.ipynb index 9588ac9e3..fde0acbae 100644 --- a/doc/pub/week44/ipynb/week44.ipynb +++ b/doc/pub/week44/ipynb/week44.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "4a1097ac", + "id": "505a71a4", "metadata": { "editable": true }, @@ -14,7 +14,7 @@ }, { "cell_type": "markdown", - "id": "efc949e8", + "id": "2b39d5bf", "metadata": { "editable": true }, @@ -22,14 +22,14 @@ "# Week 44: Decision Trees, Ensemble methods and Random Forests\n", "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n", "\n", - "Date: **Nov 3, 2022**\n", + "Date: **Nov 4, 2022**\n", "\n", "Copyright 1999-2022, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license" ] }, { "cell_type": "markdown", - "id": "c56693b1", + "id": "c9fdbceb", "metadata": { "editable": true }, @@ -55,7 +55,7 @@ }, { "cell_type": "markdown", - "id": "d15b5fdb", + "id": "ef1cbff2", "metadata": { "editable": true }, @@ -73,7 +73,7 @@ }, { "cell_type": "markdown", - "id": "ff2cff16", + "id": "157ba0f6", "metadata": { "editable": true }, @@ -104,7 +104,7 @@ }, { "cell_type": "markdown", - "id": "21a1867d", + "id": "555a4096", "metadata": { "editable": true }, @@ -124,7 +124,7 @@ }, { "cell_type": "markdown", - "id": "8c897a0b", + "id": "098da09d", "metadata": { "editable": true }, @@ -138,7 +138,7 @@ }, { "cell_type": "markdown", - "id": "69606bb7", + "id": "a24e1fc4", "metadata": { "editable": true }, @@ -151,7 +151,7 @@ }, { "cell_type": "markdown", - "id": "c0cfad4d", + "id": "d17c8876", "metadata": { "editable": true }, @@ -169,7 +169,7 @@ }, { "cell_type": "markdown", - "id": "1de57e29", + "id": "cd88ebe6", "metadata": { "editable": true }, @@ -192,7 +192,7 @@ }, { "cell_type": "markdown", - "id": "1e0a7b02", + "id": "4ad8a096", "metadata": { "editable": true }, @@ -215,7 +215,7 @@ }, { "cell_type": "markdown", - "id": "bc5c3d60", + "id": "139da320", "metadata": { "editable": true }, @@ -226,7 +226,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "224951d6", + "id": "dad5889b", "metadata": { "collapsed": false, "editable": true @@ -327,7 +327,7 @@ }, { "cell_type": "markdown", - "id": "9a9c7131", + "id": "b35c5b7b", "metadata": { "editable": true }, @@ -349,7 +349,7 @@ }, { "cell_type": "markdown", - "id": "f61f08f2", + "id": "6af12ad1", "metadata": { "editable": true }, @@ -361,7 +361,7 @@ }, { "cell_type": "markdown", - "id": "d55d09fb", + "id": "5a3c6ddf", "metadata": { "editable": true }, @@ -372,7 +372,7 @@ }, { "cell_type": "markdown", - "id": "e5fc9d2f", + "id": "7af28e13", "metadata": { "editable": true }, @@ -394,7 +394,7 @@ }, { "cell_type": "markdown", - "id": "b0b79e87", + "id": "74d2083e", "metadata": { "editable": true }, @@ -407,7 +407,7 @@ }, { "cell_type": "markdown", - "id": "b202f606", + "id": "1e8dd4e4", "metadata": { "editable": true }, @@ -419,7 +419,7 @@ }, { "cell_type": "markdown", - "id": "d264403d", + "id": "c713fb82", "metadata": { "editable": true }, @@ -429,7 +429,7 @@ }, { "cell_type": "markdown", - "id": "e7ff63dd", + "id": "79686e95", "metadata": { "editable": true }, @@ -441,7 +441,7 @@ }, { "cell_type": "markdown", - "id": "137e2a4b", + "id": "77a50747", "metadata": { "editable": true }, @@ -451,7 +451,7 @@ }, { "cell_type": "markdown", - "id": "eb2f307a", + "id": "c412345b", "metadata": { "editable": true }, @@ -463,7 +463,7 @@ }, { "cell_type": "markdown", - "id": "b9ae9b3d", + "id": "03b3969f", "metadata": { "editable": true }, @@ -496,7 +496,7 @@ }, { "cell_type": "markdown", - "id": "1d590bd6", + "id": "d2ae75bc", "metadata": { "editable": true }, @@ -520,7 +520,7 @@ }, { "cell_type": "markdown", - "id": "527d4779", + "id": "6fa1db1f", "metadata": { "editable": true }, @@ -532,7 +532,7 @@ }, { "cell_type": "markdown", - "id": "c417e43e", + "id": "27d0ab31", "metadata": { "editable": true }, @@ -544,7 +544,7 @@ }, { "cell_type": "markdown", - "id": "3ee9c91e", + "id": "ec453b01", "metadata": { "editable": true }, @@ -572,7 +572,7 @@ }, { "cell_type": "markdown", - "id": "080d4623", + "id": "6ad7b798", "metadata": { "editable": true }, @@ -598,7 +598,7 @@ }, { "cell_type": "markdown", - "id": "5435eb16", + "id": "ca53429b", "metadata": { "editable": true }, @@ -621,7 +621,7 @@ }, { "cell_type": "markdown", - "id": "0595fe67", + "id": "b0bc4c02", "metadata": { "editable": true }, @@ -648,7 +648,7 @@ }, { "cell_type": "markdown", - "id": "96bf0c71", + "id": "30c2734e", "metadata": { "editable": true }, @@ -667,7 +667,7 @@ }, { "cell_type": "markdown", - "id": "252dba5d", + "id": "d601c91d", "metadata": { "editable": true }, @@ -679,7 +679,7 @@ }, { "cell_type": "markdown", - "id": "c9b3e616", + "id": "ff524671", "metadata": { "editable": true }, @@ -692,7 +692,7 @@ }, { "cell_type": "markdown", - "id": "352fd0de", + "id": "f74bfe61", "metadata": { "editable": true }, @@ -704,7 +704,7 @@ }, { "cell_type": "markdown", - "id": "dfa5ce6e", + "id": "c0f5c807", "metadata": { "editable": true }, @@ -714,7 +714,7 @@ }, { "cell_type": "markdown", - "id": "16b7a41e", + "id": "cf055e39", "metadata": { "editable": true }, @@ -726,7 +726,7 @@ }, { "cell_type": "markdown", - "id": "a45ff7d2", + "id": "1750d116", "metadata": { "editable": true }, @@ -736,7 +736,7 @@ }, { "cell_type": "markdown", - "id": "2b26898d", + "id": "62509c8a", "metadata": { "editable": true }, @@ -748,7 +748,7 @@ }, { "cell_type": "markdown", - "id": "43dfac8f", + "id": "37085014", "metadata": { "editable": true }, @@ -759,7 +759,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "ac2b7f33", + "id": "117f996b", "metadata": { "collapsed": false, "editable": true @@ -803,7 +803,7 @@ }, { "cell_type": "markdown", - "id": "cfef7a15", + "id": "fa1917f8", "metadata": { "editable": true }, @@ -814,7 +814,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "d520b13d", + "id": "47524be8", "metadata": { "collapsed": false, "editable": true @@ -849,7 +849,7 @@ }, { "cell_type": "markdown", - "id": "68362903", + "id": "b054b690", "metadata": { "editable": true }, @@ -862,7 +862,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "fcb2e90e", + "id": "74fba157", "metadata": { "collapsed": false, "editable": true @@ -880,7 +880,7 @@ }, { "cell_type": "markdown", - "id": "58884e85", + "id": "8afb1641", "metadata": { "editable": true }, @@ -894,7 +894,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "b04fce7a", + "id": "47c60687", "metadata": { "collapsed": false, "editable": true @@ -913,7 +913,7 @@ }, { "cell_type": "markdown", - "id": "b710a7bf", + "id": "e3d4eabc", "metadata": { "editable": true }, @@ -933,7 +933,7 @@ }, { "cell_type": "markdown", - "id": "ede7746c", + "id": "4705bc8b", "metadata": { "editable": true }, @@ -950,7 +950,7 @@ }, { "cell_type": "markdown", - "id": "e4e70f8f", + "id": "5e18b708", "metadata": { "editable": true }, @@ -962,7 +962,7 @@ }, { "cell_type": "markdown", - "id": "c9b1c409", + "id": "5cdf02aa", "metadata": { "editable": true }, @@ -979,7 +979,7 @@ }, { "cell_type": "markdown", - "id": "f23b1788", + "id": "ae0e7e28", "metadata": { "editable": true }, @@ -992,7 +992,7 @@ }, { "cell_type": "markdown", - "id": "c7796ff8", + "id": "8f45de6f", "metadata": { "editable": true }, @@ -1004,7 +1004,7 @@ }, { "cell_type": "markdown", - "id": "deac0954", + "id": "17b99090", "metadata": { "editable": true }, @@ -1014,7 +1014,7 @@ }, { "cell_type": "markdown", - "id": "f73580ea", + "id": "3cbc11c1", "metadata": { "editable": true }, @@ -1026,7 +1026,7 @@ }, { "cell_type": "markdown", - "id": "6704c8d5", + "id": "b1f715a7", "metadata": { "editable": true }, @@ -1036,7 +1036,7 @@ }, { "cell_type": "markdown", - "id": "5d6140f5", + "id": "2458e3be", "metadata": { "editable": true }, @@ -1048,7 +1048,7 @@ }, { "cell_type": "markdown", - "id": "1173e734", + "id": "053d5afe", "metadata": { "editable": true }, @@ -1061,7 +1061,7 @@ }, { "cell_type": "markdown", - "id": "f4672604", + "id": "6df42d53", "metadata": { "editable": true }, @@ -1076,7 +1076,7 @@ }, { "cell_type": "markdown", - "id": "8a44ec0e", + "id": "1f4dd829", "metadata": { "editable": true }, @@ -1102,7 +1102,7 @@ }, { "cell_type": "markdown", - "id": "33ebc8eb", + "id": "51d24dc0", "metadata": { "editable": true }, @@ -1130,7 +1130,7 @@ }, { "cell_type": "markdown", - "id": "46b3aed1", + "id": "bc245165", "metadata": { "editable": true }, @@ -1147,7 +1147,7 @@ }, { "cell_type": "markdown", - "id": "2f28fbc3", + "id": "6b1a42ac", "metadata": { "editable": true }, @@ -1161,7 +1161,7 @@ }, { "cell_type": "markdown", - "id": "7a671f69", + "id": "908fd5d1", "metadata": { "editable": true }, @@ -1177,7 +1177,7 @@ }, { "cell_type": "markdown", - "id": "367b0c69", + "id": "d47dd6cd", "metadata": { "editable": true }, @@ -1188,7 +1188,7 @@ { "cell_type": "code", "execution_count": 6, - "id": "06147cf3", + "id": "4a166783", "metadata": { "collapsed": false, "editable": true @@ -1259,7 +1259,7 @@ }, { "cell_type": "markdown", - "id": "9fa00dec", + "id": "ffb9ed8f", "metadata": { "editable": true }, @@ -1303,7 +1303,7 @@ }, { "cell_type": "markdown", - "id": "4e5121fe", + "id": "1c83fd43", "metadata": { "editable": true }, @@ -1314,7 +1314,7 @@ { "cell_type": "code", "execution_count": 7, - "id": "142bd93b", + "id": "08a41c5b", "metadata": { "collapsed": false, "editable": true @@ -1392,7 +1392,7 @@ }, { "cell_type": "markdown", - "id": "64dab0e6", + "id": "d8be614e", "metadata": { "editable": true }, @@ -1410,7 +1410,7 @@ { "cell_type": "code", "execution_count": 8, - "id": "b87f7791", + "id": "72cc4a3e", "metadata": { "collapsed": false, "editable": true @@ -1481,7 +1481,7 @@ }, { "cell_type": "markdown", - "id": "5e7dc45e", + "id": "2b3eea9e", "metadata": { "editable": true }, @@ -1492,7 +1492,7 @@ { "cell_type": "code", "execution_count": 9, - "id": "4a13a11f", + "id": "41bc1f8a", "metadata": { "collapsed": false, "editable": true @@ -1567,7 +1567,7 @@ }, { "cell_type": "markdown", - "id": "3efdfac4", + "id": "606fcff7", "metadata": { "editable": true }, @@ -1578,7 +1578,7 @@ { "cell_type": "code", "execution_count": 10, - "id": "6c7deb0b", + "id": "e888b633", "metadata": { "collapsed": false, "editable": true @@ -1609,7 +1609,7 @@ }, { "cell_type": "markdown", - "id": "1d5a8fad", + "id": "0135d383", "metadata": { "editable": true }, @@ -1620,7 +1620,7 @@ { "cell_type": "code", "execution_count": 11, - "id": "f44c620b", + "id": "6819c3dc", "metadata": { "collapsed": false, "editable": true @@ -1638,7 +1638,7 @@ { "cell_type": "code", "execution_count": 12, - "id": "17618b84", + "id": "6f0b051c", "metadata": { "collapsed": false, "editable": true @@ -1653,7 +1653,7 @@ }, { "cell_type": "markdown", - "id": "5c2bb634", + "id": "7aed2b68", "metadata": { "editable": true }, @@ -1664,7 +1664,7 @@ { "cell_type": "code", "execution_count": 13, - "id": "30db4227", + "id": "286c6bbc", "metadata": { "collapsed": false, "editable": true @@ -1714,7 +1714,7 @@ { "cell_type": "code", "execution_count": 14, - "id": "329cbca4", + "id": "e62f6e67", "metadata": { "collapsed": false, "editable": true @@ -1753,7 +1753,7 @@ }, { "cell_type": "markdown", - "id": "0183baf5", + "id": "4e727c9a", "metadata": { "editable": true }, @@ -1777,7 +1777,7 @@ }, { "cell_type": "markdown", - "id": "51163190", + "id": "94aee58b", "metadata": { "editable": true }, @@ -1805,7 +1805,7 @@ }, { "cell_type": "markdown", - "id": "98ee1800", + "id": "fa218024", "metadata": { "editable": true }, @@ -1836,7 +1836,7 @@ }, { "cell_type": "markdown", - "id": "b71e6020", + "id": "150083ba", "metadata": { "editable": true }, @@ -1852,7 +1852,7 @@ }, { "cell_type": "markdown", - "id": "7f22f3ab", + "id": "92509128", "metadata": { "editable": true }, @@ -1876,7 +1876,7 @@ }, { "cell_type": "markdown", - "id": "ad9da94b", + "id": "131e117d", "metadata": { "editable": true }, @@ -1907,7 +1907,7 @@ }, { "cell_type": "markdown", - "id": "184cbf97", + "id": "1ed2934c", "metadata": { "editable": true }, @@ -1918,7 +1918,7 @@ { "cell_type": "code", "execution_count": 15, - "id": "90504a44", + "id": "507dcf9e", "metadata": { "collapsed": false, "editable": true @@ -1966,7 +1966,7 @@ }, { "cell_type": "markdown", - "id": "b5f6a377", + "id": "fe8dbd6a", "metadata": { "editable": true }, @@ -1977,7 +1977,7 @@ { "cell_type": "code", "execution_count": 16, - "id": "6057fd9e", + "id": "d2ae28bc", "metadata": { "collapsed": false, "editable": true @@ -2011,7 +2011,7 @@ }, { "cell_type": "markdown", - "id": "cd19a241", + "id": "d8eb488a", "metadata": { "editable": true }, @@ -2024,7 +2024,7 @@ { "cell_type": "code", "execution_count": 17, - "id": "7febdfe5", + "id": "77e9aba3", "metadata": { "collapsed": false, "editable": true @@ -2077,7 +2077,7 @@ }, { "cell_type": "markdown", - "id": "8bd7cc8e", + "id": "bdf5b8f2", "metadata": { "editable": true }, @@ -2088,7 +2088,7 @@ { "cell_type": "code", "execution_count": 18, - "id": "88021894", + "id": "235549bb", "metadata": { "collapsed": false, "editable": true @@ -2118,7 +2118,7 @@ { "cell_type": "code", "execution_count": 19, - "id": "346c9181", + "id": "3b1d55b5", "metadata": { "collapsed": false, "editable": true @@ -2136,7 +2136,7 @@ { "cell_type": "code", "execution_count": 20, - "id": "d1c812b1", + "id": "cd9cde74", "metadata": { "collapsed": false, "editable": true @@ -2156,7 +2156,7 @@ { "cell_type": "code", "execution_count": 21, - "id": "8fe89d33", + "id": "dae17212", "metadata": { "collapsed": false, "editable": true @@ -2173,7 +2173,7 @@ }, { "cell_type": "markdown", - "id": "da68429c", + "id": "85b65a3d", "metadata": { "editable": true }, @@ -2195,7 +2195,7 @@ }, { "cell_type": "markdown", - "id": "dc0a9f7f", + "id": "794932d0", "metadata": { "editable": true }, @@ -2227,7 +2227,7 @@ }, { "cell_type": "markdown", - "id": "b898d9d8", + "id": "82788103", "metadata": { "editable": true }, @@ -2241,7 +2241,7 @@ { "cell_type": "code", "execution_count": 22, - "id": "1d7eddbd", + "id": "322b1775", "metadata": { "collapsed": false, "editable": true @@ -2310,7 +2310,7 @@ }, { "cell_type": "markdown", - "id": "a9fc4116", + "id": "1e6499a0", "metadata": { "editable": true }, @@ -2333,7 +2333,7 @@ }, { "cell_type": "markdown", - "id": "5aba126e", + "id": "2486f4b5", "metadata": { "editable": true }, @@ -2345,7 +2345,7 @@ }, { "cell_type": "markdown", - "id": "317eca6a", + "id": "d1bd8ffa", "metadata": { "editable": true }, @@ -2370,7 +2370,7 @@ }, { "cell_type": "markdown", - "id": "98bcf577", + "id": "e2554d08", "metadata": { "editable": true }, @@ -2396,7 +2396,7 @@ }, { "cell_type": "markdown", - "id": "15893a56", + "id": "a0ccffcb", "metadata": { "editable": true }, @@ -2407,7 +2407,7 @@ { "cell_type": "code", "execution_count": 23, - "id": "6bb7a5bb", + "id": "b6fdf6c9", "metadata": { "collapsed": false, "editable": true @@ -2479,7 +2479,7 @@ }, { "cell_type": "markdown", - "id": "444d35db", + "id": "5a2f9525", "metadata": { "editable": true }, @@ -2495,7 +2495,7 @@ }, { "cell_type": "markdown", - "id": "7d667059", + "id": "f78ffcef", "metadata": { "editable": true }, @@ -2506,7 +2506,7 @@ { "cell_type": "code", "execution_count": 24, - "id": "5cdd26a2", + "id": "be875c48", "metadata": { "collapsed": false, "editable": true @@ -2521,7 +2521,7 @@ { "cell_type": "code", "execution_count": 25, - "id": "8ced2ac9", + "id": "500f4e34", "metadata": { "collapsed": false, "editable": true @@ -2536,6 +2536,760 @@ "y_pred_rf = rnd_clf.predict(X_test)\n", "np.sum(y_pred == y_pred_rf) / len(y_pred)" ] + }, + { + "cell_type": "markdown", + "id": "df4d815c", + "metadata": { + "editable": true + }, + "source": [ + "## Boosting, a Bird's Eye View\n", + "\n", + "The basic idea is to combine weak classifiers in order to create a good\n", + "classifier. With a weak classifier we often intend a classifier which\n", + "produces results which are only slightly better than we would get by\n", + "random guesses.\n", + "\n", + "This is done by applying in an iterative way a weak (or a standard\n", + "classifier like decision trees) to modify the data. In each iteration\n", + "we emphasize those observations which are misclassified by weighting\n", + "them with a factor." + ] + }, + { + "cell_type": "markdown", + "id": "c361d0e5", + "metadata": { + "editable": true + }, + "source": [ + "## What is boosting? Additive Modelling/Iterative Fitting\n", + "\n", + "Boosting is a way of fitting an additive expansion in a set of\n", + "elementary basis functions like for example some simple polynomials.\n", + "Assume for example that we have a function" + ] + }, + { + "cell_type": "markdown", + "id": "7110270e", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "f_M(x) = \\sum_{i=1}^M \\beta_m b(x;\\gamma_m),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "d6868ef4", + "metadata": { + "editable": true + }, + "source": [ + "where $\\beta_m$ are the expansion parameters to be determined in a\n", + "minimization process and $b(x;\\gamma_m)$ are some simple functions of\n", + "the multivariable parameter $x$ which is characterized by the\n", + "parameters $\\gamma_m$.\n", + "\n", + "As an example, consider the Sigmoid function we used in logistic\n", + "regression. In that case, we can translate the function\n", + "$b(x;\\gamma_m)$ into the Sigmoid function" + ] + }, + { + "cell_type": "markdown", + "id": "a47ff3f6", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\sigma(t) = \\frac{1}{1+\\exp{(-t)}},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "9f1c7bcf", + "metadata": { + "editable": true + }, + "source": [ + "where $t=\\gamma_0+\\gamma_1 x$ and the parameters $\\gamma_0$ and\n", + "$\\gamma_1$ were determined by the Logistic Regression fitting\n", + "algorithm.\n", + "\n", + "As another example, consider the cost function we defined for linear regression" + ] + }, + { + "cell_type": "markdown", + "id": "fdd953a4", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\boldsymbol{y},\\boldsymbol{f}) = \\frac{1}{n} \\sum_{i=0}^{n-1}(y_i-f(x_i))^2.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "8d25b4f3", + "metadata": { + "editable": true + }, + "source": [ + "In this case the function $f(x)$ was replaced by the design matrix\n", + "$\\boldsymbol{X}$ and the unknown linear regression parameters $\\boldsymbol{\\beta}$,\n", + "that is $\\boldsymbol{f}=\\boldsymbol{X}\\boldsymbol{\\beta}$. In linear regression we can \n", + "simply invert a matrix and obtain the parameters $\\beta$ by" + ] + }, + { + "cell_type": "markdown", + "id": "a62fee9a", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{\\beta}=\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "c83178c1", + "metadata": { + "editable": true + }, + "source": [ + "In iterative fitting or additive modeling, we minimize the cost function with respect to the parameters $\\beta_m$ and $\\gamma_m$." + ] + }, + { + "cell_type": "markdown", + "id": "a21370ef", + "metadata": { + "editable": true + }, + "source": [ + "## Iterative Fitting, Regression and Squared-error Cost Function\n", + "\n", + "The way we proceed is as follows (here we specialize to the squared-error cost function)\n", + "\n", + "1. Establish a cost function, here ${\\cal C}(\\boldsymbol{y},\\boldsymbol{f}) = \\frac{1}{n} \\sum_{i=0}^{n-1}(y_i-f_M(x_i))^2$ with $f_M(x) = \\sum_{i=1}^M \\beta_m b(x;\\gamma_m)$.\n", + "\n", + "2. Initialize with a guess $f_0(x)$. It could be one or even zero or some random numbers.\n", + "\n", + "3. For $m=1:M$\n", + "\n", + "a. minimize $\\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\\beta b(x;\\gamma))^2$ wrt $\\gamma$ and $\\beta$\n", + "\n", + "b. This gives the optimal values $\\beta_m$ and $\\gamma_m$\n", + "\n", + "c. Determine then the new values $f_m(x)=f_{m-1}(x) +\\beta_m b(x;\\gamma_m)$\n", + "\n", + "We could use any of the algorithms we have discussed till now. If we\n", + "use trees, $\\gamma$ parameterizes the split variables and split points\n", + "at the internal nodes, and the predictions at the terminal nodes." + ] + }, + { + "cell_type": "markdown", + "id": "6de36827", + "metadata": { + "editable": true + }, + "source": [ + "## Squared-Error Example and Iterative Fitting\n", + "\n", + "To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function.\n", + "\n", + "For simplicity we assume also that our functions $b(x;\\gamma)=1+\\gamma x$. \n", + "\n", + "This means that for every iteration $m$, we need to optimize" + ] + }, + { + "cell_type": "markdown", + "id": "456fc000", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "(\\beta_m,\\gamma_m) = \\mathrm{argmin}_{\\beta,\\lambda}\\hspace{0.1cm} \\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\\beta b(x;\\gamma))^2=\\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\\beta(1+\\gamma x_i))^2.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "daad2bc0", + "metadata": { + "editable": true + }, + "source": [ + "We start our iteration by simply setting $f_0(x)=0$. \n", + "Taking the derivatives with respect to $\\beta$ and $\\gamma$ we obtain" + ] + }, + { + "cell_type": "markdown", + "id": "87835910", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\frac{\\partial {\\cal C}}{\\partial \\beta} = -2\\sum_{i}(1+\\gamma x_i)(y_i-\\beta(1+\\gamma x_i))=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "6c899f4d", + "metadata": { + "editable": true + }, + "source": [ + "and" + ] + }, + { + "cell_type": "markdown", + "id": "7527f53a", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\frac{\\partial {\\cal C}}{\\partial \\gamma} =-2\\sum_{i}\\beta x_i(y_i-\\beta(1+\\gamma x_i))=0.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "2ac6036b", + "metadata": { + "editable": true + }, + "source": [ + "We can then rewrite these equations as (defining $\\boldsymbol{w}=\\boldsymbol{e}+\\gamma \\boldsymbol{x})$ with $\\boldsymbol{e}$ being the unit vector)" + ] + }, + { + "cell_type": "markdown", + "id": "95c07abf", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\gamma \\boldsymbol{w}^T(\\boldsymbol{y}-\\beta\\gamma \\boldsymbol{w})=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "118865e3", + "metadata": { + "editable": true + }, + "source": [ + "which gives us $\\beta = \\boldsymbol{w}^T\\boldsymbol{y}/(\\boldsymbol{w}^T\\boldsymbol{w})$. Similarly we have" + ] + }, + { + "cell_type": "markdown", + "id": "682ffd18", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\beta\\gamma \\boldsymbol{x}^T(\\boldsymbol{y}-\\beta(1+\\gamma \\boldsymbol{x}))=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "0e3677f8", + "metadata": { + "editable": true + }, + "source": [ + "which leads to $\\gamma =(\\boldsymbol{x}^T\\boldsymbol{y}-\\beta\\boldsymbol{x}^T\\boldsymbol{e})/(\\beta\\boldsymbol{x}^T\\boldsymbol{x})$. Inserting\n", + "for $\\beta$ gives us an equation for $\\gamma$. This is a non-linear equation in the unknown $\\gamma$ and has to be solved numerically. \n", + "\n", + "The solution to these two equations gives us in turn $\\beta_1$ and $\\gamma_1$ leading to the new expression for $f_1(x)$ as\n", + "$f_1(x) = \\beta_1(1+\\gamma_1x)$. Doing this $M$ times results in our final estimate for the function $f$." + ] + }, + { + "cell_type": "markdown", + "id": "8b156288", + "metadata": { + "editable": true + }, + "source": [ + "## Iterative Fitting, Classification and AdaBoost\n", + "\n", + "Let us consider a binary classification problem with two outcomes $y_i \\in \\{-1,1\\}$ and $i=0,1,2,\\dots,n-1$ as our set of\n", + "observations. We define a classification function $G(x)$ which produces a prediction taking one or the other of the two values \n", + "$\\{-1,1\\}$.\n", + "\n", + "The error rate of the training sample is then" + ] + }, + { + "cell_type": "markdown", + "id": "f5e80b48", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathrm{\\overline{err}}=\\frac{1}{n} \\sum_{i=0}^{n-1} I(y_i\\ne G(x_i)).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "b13a5b0e", + "metadata": { + "editable": true + }, + "source": [ + "The iterative procedure starts with defining a weak classifier whose\n", + "error rate is barely better than random guessing. The iterative\n", + "procedure in boosting is to sequentially apply a weak\n", + "classification algorithm to repeatedly modified versions of the data\n", + "producing a sequence of weak classifiers $G_m(x)$.\n", + "\n", + "Here we will express our function $f(x)$ in terms of $G(x)$. That is" + ] + }, + { + "cell_type": "markdown", + "id": "5d2f7067", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "f_M(x) = \\sum_{i=1}^M \\beta_m b(x;\\gamma_m),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "5df4fa6b", + "metadata": { + "editable": true + }, + "source": [ + "will be a function of" + ] + }, + { + "cell_type": "markdown", + "id": "a49c7c9d", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "G_M(x) = \\mathrm{sign} \\sum_{i=1}^M \\alpha_m G_m(x).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "8b9b271e", + "metadata": { + "editable": true + }, + "source": [ + "## Adaptive Boosting, AdaBoost\n", + "\n", + "In our iterative procedure we define thus" + ] + }, + { + "cell_type": "markdown", + "id": "72458577", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "f_m(x) = f_{m-1}(x)+\\beta_mG_m(x).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "29b53f28", + "metadata": { + "editable": true + }, + "source": [ + "The simplest possible cost function which leads (also simple from a computational point of view) to the AdaBoost algorithm is the\n", + "exponential cost/loss function defined as" + ] + }, + { + "cell_type": "markdown", + "id": "24432692", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\boldsymbol{y},\\boldsymbol{f}) = \\sum_{i=0}^{n-1}\\exp{(-y_i(f_{m-1}(x_i)+\\beta G(x_i))}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "80e7fe8c", + "metadata": { + "editable": true + }, + "source": [ + "We optimize $\\beta$ and $G$ for each value of $m=1:M$ as we did in the regression case.\n", + "This is normally done in two steps. Let us however first rewrite the cost function as" + ] + }, + { + "cell_type": "markdown", + "id": "3a21953c", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\boldsymbol{y},\\boldsymbol{f}) = \\sum_{i=0}^{n-1}w_i^{m}\\exp{(-y_i\\beta G(x_i))},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "32c89894", + "metadata": { + "editable": true + }, + "source": [ + "where we have defined $w_i^m= \\exp{(-y_if_{m-1}(x_i))}$." + ] + }, + { + "cell_type": "markdown", + "id": "86d9d370", + "metadata": { + "editable": true + }, + "source": [ + "## Building up AdaBoost\n", + "\n", + "First, for any $\\beta > 0$, we optimize $G$ by setting" + ] + }, + { + "cell_type": "markdown", + "id": "2a2c6cee", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "G_m(x) = \\mathrm{sign} \\sum_{i=0}^{n-1} w_i^m I(y_i \\ne G_(x_i)),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "fd52af97", + "metadata": { + "editable": true + }, + "source": [ + "which is the classifier that minimizes the weighted error rate in predicting $y$.\n", + "\n", + "We can do this by rewriting" + ] + }, + { + "cell_type": "markdown", + "id": "910099ac", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\exp{-(\\beta)}\\sum_{y_i=G(x_i)}w_i^m+\\exp{(\\beta)}\\sum_{y_i\\ne G(x_i)}w_i^m,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4ccdca71", + "metadata": { + "editable": true + }, + "source": [ + "which can be rewritten as" + ] + }, + { + "cell_type": "markdown", + "id": "e0d1fcfa", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "(\\exp{(\\beta)}-\\exp{-(\\beta)})\\sum_{i=0}^{n-1}w_i^mI(y_i\\ne G(x_i))+\\exp{(-\\beta)}\\sum_{i=0}^{n-1}w_i^m=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "01142bd3", + "metadata": { + "editable": true + }, + "source": [ + "which leads to" + ] + }, + { + "cell_type": "markdown", + "id": "d00ed408", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\beta_m = \\frac{1}{2}\\log{\\frac{1-\\mathrm{\\overline{err}}}{\\mathrm{\\overline{err}}}},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "1488e21c", + "metadata": { + "editable": true + }, + "source": [ + "where we have redefined the error as" + ] + }, + { + "cell_type": "markdown", + "id": "685a3c45", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathrm{\\overline{err}}_m=\\frac{1}{n}\\frac{\\sum_{i=0}^{n-1}w_i^mI(y_i\\ne G(x_i)}{\\sum_{i=0}^{n-1}w_i^m},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f001ece2", + "metadata": { + "editable": true + }, + "source": [ + "which leads to an update of" + ] + }, + { + "cell_type": "markdown", + "id": "1669bf49", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "f_m(x) = f_{m-1}(x) +\\beta_m G_m(x).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "b80cc4a1", + "metadata": { + "editable": true + }, + "source": [ + "This leads to the new weights" + ] + }, + { + "cell_type": "markdown", + "id": "4b8b5bce", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "w_i^{m+1} = w_i^m \\exp{(-y_i\\beta_m G_m(x_i))}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "186fdada", + "metadata": { + "editable": true + }, + "source": [ + "## Adaptive boosting: AdaBoost, Basic Algorithm\n", + "\n", + "The algorithm here is rather straightforward. Assume that our weak\n", + "classifier is a decision tree and we consider a binary set of outputs\n", + "with $y_i \\in \\{-1,1\\}$ and $i=0,1,2,\\dots,n-1$ as our set of\n", + "observations. Our design matrix is given in terms of the\n", + "feature/predictor vectors\n", + "$\\boldsymbol{X}=[\\boldsymbol{x}_0\\boldsymbol{x}_1\\dots\\boldsymbol{x}_{p-1}]$. Finally, we define also a\n", + "classifier determined by our data via a function $G(x)$. This function tells us how well we are able to classify our outputs/targets $\\boldsymbol{y}$. \n", + "\n", + "We have already defined the misclassification error $\\mathrm{err}$ as" + ] + }, + { + "cell_type": "markdown", + "id": "12d25569", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathrm{err}=\\frac{1}{n}\\sum_{i=0}^{n-1}I(y_i\\ne G(x_i)),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "cc8766d0", + "metadata": { + "editable": true + }, + "source": [ + "where the function $I()$ is one if we misclassify and zero if we classify correctly." + ] + }, + { + "cell_type": "markdown", + "id": "fd830c99", + "metadata": { + "editable": true + }, + "source": [ + "## Basic Steps of AdaBoost\n", + "\n", + "With the above definitions we are now ready to set up the algorithm for AdaBoost.\n", + "The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases.\n", + "1. We start by initializing all weights to $w_i = 1/n$, with $i=0,1,2,\\dots n-1$. It is easy to see that we must have $\\sum_{i=0}^{n-1}w_i = 1$.\n", + "\n", + "2. We rewrite the misclassification error as" + ] + }, + { + "cell_type": "markdown", + "id": "2ada374a", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathrm{\\overline{err}}_m=\\frac{\\sum_{i=0}^{n-1}w_i^m I(y_i\\ne G(x_i))}{\\sum_{i=0}^{n-1}w_i},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "82d37d21", + "metadata": { + "editable": true + }, + "source": [ + "1. Then we start looping over all attempts at classifying, namely we start an iterative process for $m=1:M$, where $M$ is the final number of classifications. Our given classifier could for example be a plain decision tree.\n", + "\n", + "a. Fit then a given classifier to the training set using the weights $w_i$.\n", + "\n", + "b. Compute then $\\mathrm{err}$ and figure out which events are classified properly and which are classified wrongly.\n", + "\n", + "c. Define a quantity $\\alpha_{m} = \\log{(1-\\mathrm{\\overline{err}}_m)/\\mathrm{\\overline{err}}_m}$\n", + "\n", + "d. Set the new weights to $w_i = w_i\\times \\exp{(\\alpha_m I(y_i\\ne G(x_i)}$.\n", + "\n", + "5. Compute the new classifier $G(x)= \\sum_{i=0}^{n-1}\\alpha_m I(y_i\\ne G(x_i)$.\n", + "\n", + "For the iterations with $m \\le 2$ the weights are modified\n", + "individually at each steps. The observations which were misclassified\n", + "at iteration $m-1$ have a weight which is larger than those which were\n", + "classified properly. As this proceeds, the observations which were\n", + "difficult to classifiy correctly are given a larger influence. Each\n", + "new classification step $m$ is then forced to concentrate on those\n", + "observations that are missed in the previous iterations." + ] + }, + { + "cell_type": "markdown", + "id": "64c328ea", + "metadata": { + "editable": true + }, + "source": [ + "## AdaBoost Examples\n", + "\n", + "Using **Scikit-Learn** it is easy to apply the adaptive boosting algorithm, as done here." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "a34612d9", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from sklearn.ensemble import AdaBoostClassifier\n", + "\n", + "ada_clf = AdaBoostClassifier(\n", + " DecisionTreeClassifier(max_depth=1), n_estimators=200,\n", + " algorithm=\"SAMME.R\", learning_rate=0.5, random_state=42)\n", + "ada_clf.fit(X_train, y_train)\n", + "\n", + "from sklearn.ensemble import AdaBoostClassifier\n", + "\n", + "ada_clf = AdaBoostClassifier(\n", + " DecisionTreeClassifier(max_depth=1), n_estimators=200,\n", + " algorithm=\"SAMME.R\", learning_rate=0.5, random_state=42)\n", + "ada_clf.fit(X_train_scaled, y_train)\n", + "y_pred = ada_clf.predict(X_test_scaled)\n", + "skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)\n", + "plt.show()\n", + "y_probas = ada_clf.predict_proba(X_test_scaled)\n", + "skplt.metrics.plot_roc(y_test, y_probas)\n", + "plt.show()\n", + "skplt.metrics.plot_cumulative_gain(y_test, y_probas)\n", + "plt.show()" + ] } ], "metadata": {}, diff --git a/doc/src/week44/week44.do.txt b/doc/src/week44/week44.do.txt index 1d1756238..e6407f014 100644 --- a/doc/src/week44/week44.do.txt +++ b/doc/src/week44/week44.do.txt @@ -1692,3 +1692,328 @@ np.sum(y_pred == y_pred_rf) / len(y_pred) + + +!split +===== Boosting, a Bird's Eye View ===== + +The basic idea is to combine weak classifiers in order to create a good +classifier. With a weak classifier we often intend a classifier which +produces results which are only slightly better than we would get by +random guesses. + +This is done by applying in an iterative way a weak (or a standard +classifier like decision trees) to modify the data. In each iteration +we emphasize those observations which are misclassified by weighting +them with a factor. + + +!split +===== What is boosting? Additive Modelling/Iterative Fitting ===== + +Boosting is a way of fitting an additive expansion in a set of +elementary basis functions like for example some simple polynomials. +Assume for example that we have a function +!bt +\[ +f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m), +\] +!et + +where $\beta_m$ are the expansion parameters to be determined in a +minimization process and $b(x;\gamma_m)$ are some simple functions of +the multivariable parameter $x$ which is characterized by the +parameters $\gamma_m$. + +As an example, consider the Sigmoid function we used in logistic +regression. In that case, we can translate the function +$b(x;\gamma_m)$ into the Sigmoid function + + +!bt +\[ +\sigma(t) = \frac{1}{1+\exp{(-t)}}, +\] +!et + +where $t=\gamma_0+\gamma_1 x$ and the parameters $\gamma_0$ and +$\gamma_1$ were determined by the Logistic Regression fitting +algorithm. + +As another example, consider the cost function we defined for linear regression +!bt +\[ +C(\bm{y},\bm{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f(x_i))^2. +\] +!et + +In this case the function $f(x)$ was replaced by the design matrix +$\bm{X}$ and the unknown linear regression parameters $\bm{\beta}$, +that is $\bm{f}=\bm{X}\bm{\beta}$. In linear regression we can +simply invert a matrix and obtain the parameters $\beta$ by + +!bt +\[ +\bm{\beta}=\left(\bm{X}^T\bm{X}\right)^{-1}\bm{X}^T\bm{y}. +\] +!et + +In iterative fitting or additive modeling, we minimize the cost function with respect to the parameters $\beta_m$ and $\gamma_m$. + + +!split +===== Iterative Fitting, Regression and Squared-error Cost Function ===== + +The way we proceed is as follows (here we specialize to the squared-error cost function) + +o Establish a cost function, here ${\cal C}(\bm{y},\bm{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f_M(x_i))^2$ with $f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m)$. +o Initialize with a guess $f_0(x)$. It could be one or even zero or some random numbers. +o For $m=1:M$ + o minimize $\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2$ wrt $\gamma$ and $\beta$ + o This gives the optimal values $\beta_m$ and $\gamma_m$ + o Determine then the new values $f_m(x)=f_{m-1}(x) +\beta_m b(x;\gamma_m)$ + +We could use any of the algorithms we have discussed till now. If we +use trees, $\gamma$ parameterizes the split variables and split points +at the internal nodes, and the predictions at the terminal nodes. + + +!split +===== Squared-Error Example and Iterative Fitting ===== + +To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function. + +For simplicity we assume also that our functions $b(x;\gamma)=1+\gamma x$. + +This means that for every iteration $m$, we need to optimize + +!bt +\[ +(\beta_m,\gamma_m) = \mathrm{argmin}_{\beta,\lambda}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2=\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta(1+\gamma x_i))^2. +\] +!et + +We start our iteration by simply setting $f_0(x)=0$. +Taking the derivatives with respect to $\beta$ and $\gamma$ we obtain +!bt +\[ +\frac{\partial {\cal C}}{\partial \beta} = -2\sum_{i}(1+\gamma x_i)(y_i-\beta(1+\gamma x_i))=0, +\] +!et +and +!bt +\[ +\frac{\partial {\cal C}}{\partial \gamma} =-2\sum_{i}\beta x_i(y_i-\beta(1+\gamma x_i))=0. +\] +!et +We can then rewrite these equations as (defining $\bm{w}=\bm{e}+\gamma \bm{x})$ with $\bm{e}$ being the unit vector) +!bt +\[ +\gamma \bm{w}^T(\bm{y}-\beta\gamma \bm{w})=0, +\] +!et +which gives us $\beta = \bm{w}^T\bm{y}/(\bm{w}^T\bm{w})$. Similarly we have +!bt +\[ +\beta\gamma \bm{x}^T(\bm{y}-\beta(1+\gamma \bm{x}))=0, +\] +!et + +which leads to $\gamma =(\bm{x}^T\bm{y}-\beta\bm{x}^T\bm{e})/(\beta\bm{x}^T\bm{x})$. Inserting +for $\beta$ gives us an equation for $\gamma$. This is a non-linear equation in the unknown $\gamma$ and has to be solved numerically. + +The solution to these two equations gives us in turn $\beta_1$ and $\gamma_1$ leading to the new expression for $f_1(x)$ as +$f_1(x) = \beta_1(1+\gamma_1x)$. Doing this $M$ times results in our final estimate for the function $f$. + + + +!split +===== Iterative Fitting, Classification and AdaBoost ===== + +Let us consider a binary classification problem with two outcomes $y_i \in \{-1,1\}$ and $i=0,1,2,\dots,n-1$ as our set of +observations. We define a classification function $G(x)$ which produces a prediction taking one or the other of the two values +$\{-1,1\}$. + +The error rate of the training sample is then + +!bt +\[ +\mathrm{\overline{err}}=\frac{1}{n} \sum_{i=0}^{n-1} I(y_i\ne G(x_i)). +\] +!et + +The iterative procedure starts with defining a weak classifier whose +error rate is barely better than random guessing. The iterative +procedure in boosting is to sequentially apply a weak +classification algorithm to repeatedly modified versions of the data +producing a sequence of weak classifiers $G_m(x)$. + +Here we will express our function $f(x)$ in terms of $G(x)$. That is +!bt +\[ +f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m), +\] +!et +will be a function of +!bt +\[ +G_M(x) = \mathrm{sign} \sum_{i=1}^M \alpha_m G_m(x). +\] +!et + + + +!split +===== Adaptive Boosting, AdaBoost ===== + +In our iterative procedure we define thus +!bt +\[ +f_m(x) = f_{m-1}(x)+\beta_mG_m(x). +\] +!et + +The simplest possible cost function which leads (also simple from a computational point of view) to the AdaBoost algorithm is the +exponential cost/loss function defined as +!bt +\[ +C(\bm{y},\bm{f}) = \sum_{i=0}^{n-1}\exp{(-y_i(f_{m-1}(x_i)+\beta G(x_i))}. +\] +!et + +We optimize $\beta$ and $G$ for each value of $m=1:M$ as we did in the regression case. +This is normally done in two steps. Let us however first rewrite the cost function as + +!bt +\[ +C(\bm{y},\bm{f}) = \sum_{i=0}^{n-1}w_i^{m}\exp{(-y_i\beta G(x_i))}, +\] +!et +where we have defined $w_i^m= \exp{(-y_if_{m-1}(x_i))}$. + +!split +===== Building up AdaBoost ===== + +First, for any $\beta > 0$, we optimize $G$ by setting +!bt +\[ +G_m(x) = \mathrm{sign} \sum_{i=0}^{n-1} w_i^m I(y_i \ne G_(x_i)), +\] +!et +which is the classifier that minimizes the weighted error rate in predicting $y$. + +We can do this by rewriting +!bt +\[ +\exp{-(\beta)}\sum_{y_i=G(x_i)}w_i^m+\exp{(\beta)}\sum_{y_i\ne G(x_i)}w_i^m, +\] +!et +which can be rewritten as +!bt +\[ +(\exp{(\beta)}-\exp{-(\beta)})\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i))+\exp{(-\beta)}\sum_{i=0}^{n-1}w_i^m=0, +\] +!et +which leads to +!bt +\[ +\beta_m = \frac{1}{2}\log{\frac{1-\mathrm{\overline{err}}}{\mathrm{\overline{err}}}}, +\] +!et +where we have redefined the error as +!bt +\[ +\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i)}{\sum_{i=0}^{n-1}w_i^m}, +\] +!et +which leads to an update of +!bt +\[ +f_m(x) = f_{m-1}(x) +\beta_m G_m(x). +\] +!et +This leads to the new weights +!bt +\[ +w_i^{m+1} = w_i^m \exp{(-y_i\beta_m G_m(x_i))} +\] +!et + + +!split +===== Adaptive boosting: AdaBoost, Basic Algorithm ===== + +The algorithm here is rather straightforward. Assume that our weak +classifier is a decision tree and we consider a binary set of outputs +with $y_i \in \{-1,1\}$ and $i=0,1,2,\dots,n-1$ as our set of +observations. Our design matrix is given in terms of the +feature/predictor vectors +$\bm{X}=[\bm{x}_0\bm{x}_1\dots\bm{x}_{p-1}]$. Finally, we define also a +classifier determined by our data via a function $G(x)$. This function tells us how well we are able to classify our outputs/targets $\bm{y}$. + +We have already defined the misclassification error $\mathrm{err}$ as +!bt +\[ +\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(x_i)), +\] +!et +where the function $I()$ is one if we misclassify and zero if we classify correctly. + +!split +===== Basic Steps of AdaBoost ===== + +With the above definitions we are now ready to set up the algorithm for AdaBoost. +The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases. +o We start by initializing all weights to $w_i = 1/n$, with $i=0,1,2,\dots n-1$. It is easy to see that we must have $\sum_{i=0}^{n-1}w_i = 1$. +o We rewrite the misclassification error as +!bt +\[ +\mathrm{\overline{err}}_m=\frac{\sum_{i=0}^{n-1}w_i^m I(y_i\ne G(x_i))}{\sum_{i=0}^{n-1}w_i}, +\] +!et +o Then we start looping over all attempts at classifying, namely we start an iterative process for $m=1:M$, where $M$ is the final number of classifications. Our given classifier could for example be a plain decision tree. + o Fit then a given classifier to the training set using the weights $w_i$. + o Compute then $\mathrm{err}$ and figure out which events are classified properly and which are classified wrongly. + o Define a quantity $\alpha_{m} = \log{(1-\mathrm{\overline{err}}_m)/\mathrm{\overline{err}}_m}$ + o Set the new weights to $w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(x_i)}$. +o Compute the new classifier $G(x)= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(x_i)$. + +For the iterations with $m \le 2$ the weights are modified +individually at each steps. The observations which were misclassified +at iteration $m-1$ have a weight which is larger than those which were +classified properly. As this proceeds, the observations which were +difficult to classifiy correctly are given a larger influence. Each +new classification step $m$ is then forced to concentrate on those +observations that are missed in the previous iterations. + + + +!split +===== AdaBoost Examples ===== + +Using _Scikit-Learn_ it is easy to apply the adaptive boosting algorithm, as done here. + +!bc pycod +from sklearn.ensemble import AdaBoostClassifier + +ada_clf = AdaBoostClassifier( + DecisionTreeClassifier(max_depth=1), n_estimators=200, + algorithm="SAMME.R", learning_rate=0.5, random_state=42) +ada_clf.fit(X_train, y_train) + +from sklearn.ensemble import AdaBoostClassifier + +ada_clf = AdaBoostClassifier( + DecisionTreeClassifier(max_depth=1), n_estimators=200, + algorithm="SAMME.R", learning_rate=0.5, random_state=42) +ada_clf.fit(X_train_scaled, y_train) +y_pred = ada_clf.predict(X_test_scaled) +skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True) +plt.show() +y_probas = ada_clf.predict_proba(X_test_scaled) +skplt.metrics.plot_roc(y_test, y_probas) +plt.show() +skplt.metrics.plot_cumulative_gain(y_test, y_probas) +plt.show() +!ec +