week 45
This commit is contained in:
mhjensen
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -199,7 +212,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Oct 31, 2020</h4></center> <!-- date -->
|
||||
<center><h4>Nov 2, 2020</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
|
||||
@@ -223,7 +236,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._week45-bs008.html">9</a></li>
|
||||
<li><a href="._week45-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -183,11 +196,11 @@ MathJax.Hub.Config({
|
||||
<h2 id="___sec0" class="anchor">Overview of week 45 </h2>
|
||||
|
||||
<ul>
|
||||
<li> "Thursday: Wrapping up from last week. Bagging and Random forests.</li>
|
||||
<li> "Thursday: Wrapping up from last week. Bagging and Random forests. Boosting methods.</li>
|
||||
<li> "Friday: Boosting and gradient boosting</li>
|
||||
</ul>
|
||||
|
||||
Geron's chapter 7. See also lecture from <a href="https://www.uio.no/studier/emner/matnat/math/STK-IN4300/h20/slides/lecture_7.pdf" target="_self">STK-IN4300, lecture 7</a>. Chapter 9.2 of Hastie et al contains also a good discussion.
|
||||
Geron's chapter 7. See also lecture from <a href="https://www.uio.no/studier/emner/matnat/math/STK-IN4300/h20/slides/lecture_9.pdf" target="_self">STK-IN4300, lecture 9</a>. Chapter 10 (sections 10.1-10.10 are the most relevant ones) of Hastie et al contains also a good discussion.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -206,7 +219,7 @@ Geron's chapter 7. See also lecture from <a href="https://www.uio.no/studier/emn
|
||||
<li><a href="._week45-bs009.html">10</a></li>
|
||||
<li><a href="._week45-bs010.html">11</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs002.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -184,6 +197,8 @@ MathJax.Hub.Config({
|
||||
|
||||
<p>
|
||||
Bagging, voting and random forests.
|
||||
The material on bagging and voting is a repeat from last week and can be found in the slides from week 44.
|
||||
We repeat here the voting approach since this will serve as a motivation for boosting methods later.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -203,7 +218,7 @@ Bagging, voting and random forests.
|
||||
<li><a href="._week45-bs010.html">11</a></li>
|
||||
<li><a href="._week45-bs011.html">12</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs003.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,46 +193,25 @@ MathJax.Hub.Config({
|
||||
<a name="part0003"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec2" class="anchor">Random forests </h2>
|
||||
<h2 id="___sec2" class="anchor">Why Voting? </h2>
|
||||
|
||||
<p>
|
||||
Random forests provide an improvement over bagged trees by way of a
|
||||
small tweak that decorrelates the trees.
|
||||
The idea behind boosting, and voting as well can be phrased as follows:
|
||||
<b>Can a group of people somehow arrive at highly
|
||||
reasoned decisions, despite the weak judgement of the individual
|
||||
members?</b>
|
||||
|
||||
<p>
|
||||
As in bagging, we build a
|
||||
number of decision trees on bootstrapped training samples. But when
|
||||
building these decision trees, each time a split in a tree is
|
||||
considered, a random sample of \( m \) predictors is chosen as split
|
||||
candidates from the full set of \( p \) predictors. The split is allowed to
|
||||
use only one of those \( m \) predictors.
|
||||
The aim is to create a good classifier by combining several weak classifiers.
|
||||
<b>A weak classifier is a classifier which is able to produce results that are only slightly better than guessing at random.</b>
|
||||
|
||||
<p>
|
||||
A fresh sample of \( m \) predictors is
|
||||
taken at each split, and typically we choose
|
||||
|
||||
$$
|
||||
m\approx \sqrt{p}.
|
||||
$$
|
||||
The basic approach is to apply repeatedly (in boosting this is done in an iterative way) a weak classifier to modifications of the data.
|
||||
In voting we simply apply the law of large numbers while in boosting we give more weight to misclassified data in
|
||||
each iteration.
|
||||
|
||||
<p>
|
||||
In building a random forest, at
|
||||
each split in the tree, the algorithm is not even allowed to consider
|
||||
a majority of the available predictors.
|
||||
|
||||
<p>
|
||||
The reason for this is rather clever. Suppose that there is one very
|
||||
strong predictor in the data set, along with a number of other
|
||||
moderately strong predictors. Then in the collection of bagged
|
||||
variable importance random forest trees, most or all of the trees will
|
||||
use this strong predictor in the top split. Consequently, all of the
|
||||
bagged trees will look quite similar to each other. Hence the
|
||||
predictions from the bagged trees will be highly correlated.
|
||||
Unfortunately, averaging many highly correlated quantities does not
|
||||
lead to as large of a reduction in variance as averaging many
|
||||
uncorrelated quantities. In particular, this means that bagging will
|
||||
not lead to a substantial reduction in variance over a single tree in
|
||||
this setting.
|
||||
Decision trees play an important role as our weak classifier. They serve as the basic method.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -240,7 +232,7 @@ this setting.
|
||||
<li><a href="._week45-bs011.html">12</a></li>
|
||||
<li><a href="._week45-bs012.html">13</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs004.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,30 +193,25 @@ MathJax.Hub.Config({
|
||||
<a name="part0004"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec3" class="anchor">Random Forest Algorithm </h2>
|
||||
The algorithm described here can be applied to both classification and regression problems.
|
||||
<h2 id="___sec3" class="anchor">Tossing coins </h2>
|
||||
The simplest case is a so-called voting ensemble. To illustrate this, Think of you tossing coins with a biased outcome of 51 per cent for heads and 49% for tails.
|
||||
With only few tosses, you may not clearly see this distribution. However, after some thousands of tosses (sounds like you may have some spare time problems), there will be a clear majority of heads.
|
||||
With 2000 tosses you should see approximately 1020 heads and 980 tails.
|
||||
|
||||
<p>
|
||||
We will grow of forest of say \( B \) trees.
|
||||
We can then state that the outcome is a clear majority of heads. If you do this ten thousand times, it is easy to see that there is a 97% likelihood of a majority of heads.
|
||||
|
||||
<ol>
|
||||
<li> For \( b=1:B \)</li>
|
||||
<p>
|
||||
Another example would be to collect all polls before an
|
||||
election. Different polls may show different likelihoods for a
|
||||
candidate winning with say a majority of the popular vote. The majority vote
|
||||
would then consist in many polls indicating that this candidate will
|
||||
actually win.
|
||||
|
||||
<ul>
|
||||
<li> Draw a bootstrap sample of from the training data organized in our \( \boldsymbol{X} \) matrix.</li>
|
||||
<li> We grow then a random forest tree \( T_b \) based on the bootstrapped data by repeating the steps outlined till we reach the maximum node size is reached</li>
|
||||
|
||||
<ol>
|
||||
<li> we select \( m \le p \) variables at random from the \( p \) predictors/features</li>
|
||||
<li> pick the best split point among the \( m \) features using either the CART algorithm or the ID3 for classification and create a new node</li>
|
||||
<li> split the node into daughter nodes</li>
|
||||
</ol>
|
||||
|
||||
</ul>
|
||||
|
||||
<li> Output then the ensemble of trees \( \{T_b\}_1^{B} \) and make predictions for either a regression type of problem or a classification type of problem.</li>
|
||||
</ol>
|
||||
<p>
|
||||
The example here shows how we can implement the coin tossing case, clealry demostrating that after some tosses we see the law of large numbers kicking in.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
@@ -223,7 +231,7 @@ We will grow of forest of say \( B \) trees.
|
||||
<li><a href="._week45-bs012.html">13</a></li>
|
||||
<li><a href="._week45-bs013.html">14</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs005.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,74 +193,22 @@ MathJax.Hub.Config({
|
||||
<a name="part0005"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec4" class="anchor">Random Forests Compared with other Methods on the Cancer Data </h2>
|
||||
<h2 id="___sec4" class="anchor">Simple Voting Example, head or tail </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Load the data</span>
|
||||
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
||||
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
||||
<span style="color: #008000">print</span>(X_train<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #008000">print</span>(X_test<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
||||
logreg <span style="color: #666666">=</span> LogisticRegression(solver<span style="color: #666666">=</span><span style="color: #BA2121">'lbfgs'</span>)
|
||||
logreg<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Logistic Regression: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Support vector machine</span>
|
||||
svm <span style="color: #666666">=</span> SVC(gamma<span style="color: #666666">=</span><span style="color: #BA2121">'auto'</span>, C<span style="color: #666666">=100</span>)
|
||||
svm<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with SVM: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(svm<span style="color: #666666">.</span>score(X_test,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Decision Trees</span>
|
||||
deep_tree_clf <span style="color: #666666">=</span> DecisionTreeClassifier(max_depth<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>)
|
||||
deep_tree_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Decision Trees: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(deep_tree_clf<span style="color: #666666">.</span>score(X_test,y_test)))
|
||||
<span style="color: #408080; font-style: italic">#now scale the data</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
scaler<span style="color: #666666">.</span>fit(X_train)
|
||||
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
||||
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
||||
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
||||
logreg<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy Logistic Regression with scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Support Vector Machine</span>
|
||||
svm<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy SVM with scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Decision Trees</span>
|
||||
deep_tree_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Decision Trees and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(deep_tree_clf<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> LabelEncoder
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
|
||||
<span style="color: #408080; font-style: italic"># Data set not specificied</span>
|
||||
<span style="color: #408080; font-style: italic">#Instantiate the model with 500 trees and entropy as splitting criteria</span>
|
||||
Random_Forest_model <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=500</span>,criterion<span style="color: #666666">=</span><span style="color: #BA2121">"entropy"</span>)
|
||||
Random_Forest_model<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #408080; font-style: italic">#Cross validation</span>
|
||||
accuracy <span style="color: #666666">=</span> cross_validate(Random_Forest_model,X_test_scaled,y_test,cv<span style="color: #666666">=10</span>)[<span style="color: #BA2121">'test_score'</span>]
|
||||
<span style="color: #008000">print</span>(accuracy)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Random Forests and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(Random_Forest_model<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
y_pred <span style="color: #666666">=</span> Random_Forest_model<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
y_probas <span style="color: #666666">=</span> Random_Forest_model<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>heads_proba <span style="color: #666666">=</span> <span style="color: #666666">0.51</span>
|
||||
coin_tosses <span style="color: #666666">=</span> (np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">10000</span>, <span style="color: #666666">10</span>) <span style="color: #666666"><</span> heads_proba)<span style="color: #666666">.</span>astype(np<span style="color: #666666">.</span>int32)
|
||||
cumulative_heads_ratio <span style="color: #666666">=</span> np<span style="color: #666666">.</span>cumsum(coin_tosses, axis<span style="color: #666666">=0</span>) <span style="color: #666666">/</span> np<span style="color: #666666">.</span>arange(<span style="color: #666666">1</span>, <span style="color: #666666">10001</span>)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">8</span>,<span style="color: #666666">3.5</span>))
|
||||
plt<span style="color: #666666">.</span>plot(cumulative_heads_ratio)
|
||||
plt<span style="color: #666666">.</span>plot([<span style="color: #666666">0</span>, <span style="color: #666666">10000</span>], [<span style="color: #666666">0.51</span>, <span style="color: #666666">0.51</span>], <span style="color: #BA2121">"k--"</span>, linewidth<span style="color: #666666">=2</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"51%"</span>)
|
||||
plt<span style="color: #666666">.</span>plot([<span style="color: #666666">0</span>, <span style="color: #666666">10000</span>], [<span style="color: #666666">0.5</span>, <span style="color: #666666">0.5</span>], <span style="color: #BA2121">"k-"</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"50%"</span>)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"Number of coin tosses"</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"Heads ratio"</span>)
|
||||
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">"lower right"</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>, <span style="color: #666666">10000</span>, <span style="color: #666666">0.42</span>, <span style="color: #666666">0.58</span>])
|
||||
save_fig(<span style="color: #BA2121">"votingsimple"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
@@ -271,7 +232,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._week45-bs013.html">14</a></li>
|
||||
<li><a href="._week45-bs014.html">15</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs006.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,24 +193,55 @@ MathJax.Hub.Config({
|
||||
<a name="part0006"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec5" class="anchor">Compare Bagging on Trees with Random Forests </h2>
|
||||
<h2 id="___sec5" class="anchor">Using the Voting Classifier </h2>
|
||||
|
||||
<p>
|
||||
We can use the voting classifier on other data sets, here the excting binary case of two distinct objects using the make moons functionality of -Scikit-Learn-.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>bag_clf <span style="color: #666666">=</span> BaggingClassifier(
|
||||
DecisionTreeClassifier(splitter<span style="color: #666666">=</span><span style="color: #BA2121">"random"</span>, max_leaf_nodes<span style="color: #666666">=16</span>, random_state<span style="color: #666666">=42</span>),
|
||||
n_estimators<span style="color: #666666">=500</span>, max_samples<span style="color: #666666">=1.0</span>, bootstrap<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, n_jobs<span style="color: #666666">=-1</span>, random_state<span style="color: #666666">=42</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_moons
|
||||
|
||||
X, y <span style="color: #666666">=</span> make_moons(n_samples<span style="color: #666666">=500</span>, noise<span style="color: #666666">=0.30</span>, random_state<span style="color: #666666">=42</span>)
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(X, y, random_state<span style="color: #666666">=42</span>)
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>bag_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred <span style="color: #666666">=</span> bag_clf<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
|
||||
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=500</span>, max_leaf_nodes<span style="color: #666666">=16</span>, n_jobs<span style="color: #666666">=-1</span>, random_state<span style="color: #666666">=42</span>)
|
||||
rnd_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred_rf <span style="color: #666666">=</span> rnd_clf<span style="color: #666666">.</span>predict(X_test)
|
||||
np<span style="color: #666666">.</span>sum(y_pred <span style="color: #666666">==</span> y_pred_rf) <span style="color: #666666">/</span> <span style="color: #008000">len</span>(y_pred)
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> VotingClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
||||
|
||||
log_clf <span style="color: #666666">=</span> LogisticRegression(solver<span style="color: #666666">=</span><span style="color: #BA2121">"liblinear"</span>, random_state<span style="color: #666666">=42</span>)
|
||||
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=10</span>, random_state<span style="color: #666666">=42</span>)
|
||||
svm_clf <span style="color: #666666">=</span> SVC(gamma<span style="color: #666666">=</span><span style="color: #BA2121">"auto"</span>, random_state<span style="color: #666666">=42</span>)
|
||||
|
||||
voting_clf <span style="color: #666666">=</span> VotingClassifier(
|
||||
estimators<span style="color: #666666">=</span>[(<span style="color: #BA2121">'lr'</span>, log_clf), (<span style="color: #BA2121">'rf'</span>, rnd_clf), (<span style="color: #BA2121">'svc'</span>, svm_clf)],
|
||||
voting<span style="color: #666666">=</span><span style="color: #BA2121">'hard'</span>)
|
||||
|
||||
voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> clf <span style="color: #AA22FF; font-weight: bold">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000">print</span>(clf<span style="color: #666666">.</span><span style="color: #19177C">__class__</span><span style="color: #666666">.</span><span style="color: #19177C">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
|
||||
log_clf <span style="color: #666666">=</span> LogisticRegression(solver<span style="color: #666666">=</span><span style="color: #BA2121">"liblinear"</span>, random_state<span style="color: #666666">=42</span>)
|
||||
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=10</span>, random_state<span style="color: #666666">=42</span>)
|
||||
svm_clf <span style="color: #666666">=</span> SVC(gamma<span style="color: #666666">=</span><span style="color: #BA2121">"auto"</span>, probability<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, random_state<span style="color: #666666">=42</span>)
|
||||
voting_clf <span style="color: #666666">=</span> VotingClassifier(
|
||||
estimators<span style="color: #666666">=</span>[(<span style="color: #BA2121">'lr'</span>, log_clf), (<span style="color: #BA2121">'rf'</span>, rnd_clf), (<span style="color: #BA2121">'svc'</span>, svm_clf)],
|
||||
voting<span style="color: #666666">=</span><span style="color: #BA2121">'soft'</span>)
|
||||
voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> clf <span style="color: #AA22FF; font-weight: bold">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000">print</span>(clf<span style="color: #666666">.</span><span style="color: #19177C">__class__</span><span style="color: #666666">.</span><span style="color: #19177C">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
@@ -221,7 +265,7 @@ np<span style="color: #666666">.</span>sum(y_pred <span style="color: #666666">=
|
||||
<li><a href="._week45-bs014.html">15</a></li>
|
||||
<li><a href="._week45-bs015.html">16</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs007.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,20 +193,62 @@ MathJax.Hub.Config({
|
||||
<a name="part0007"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec6" class="anchor">Boosting, a Bird's Eye View </h2>
|
||||
<h2 id="___sec6" class="anchor">Please, not the moons again! Voting and Bagging </h2>
|
||||
|
||||
<p>
|
||||
The basic idea is to combine weak classifiers in order to create a good
|
||||
classifier. With a weak classifier we often intend a classifier which
|
||||
produces results which are only slightly better than we would get by
|
||||
random guesses.
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_moons
|
||||
|
||||
X, y <span style="color: #666666">=</span> make_moons(n_samples<span style="color: #666666">=500</span>, noise<span style="color: #666666">=0.30</span>, random_state<span style="color: #666666">=42</span>)
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(X, y, random_state<span style="color: #666666">=42</span>)
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> VotingClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
||||
|
||||
log_clf <span style="color: #666666">=</span> LogisticRegression(random_state<span style="color: #666666">=42</span>)
|
||||
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(random_state<span style="color: #666666">=42</span>)
|
||||
svm_clf <span style="color: #666666">=</span> SVC(random_state<span style="color: #666666">=42</span>)
|
||||
|
||||
voting_clf <span style="color: #666666">=</span> VotingClassifier(
|
||||
estimators<span style="color: #666666">=</span>[(<span style="color: #BA2121">'lr'</span>, log_clf), (<span style="color: #BA2121">'rf'</span>, rnd_clf), (<span style="color: #BA2121">'svc'</span>, svm_clf)],
|
||||
voting<span style="color: #666666">=</span><span style="color: #BA2121">'hard'</span>)
|
||||
voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
</pre></div>
|
||||
<p>
|
||||
This is done by applying in an iterative way a weak (or a standard
|
||||
classifier like decision trees) to modify the data. In each iteration
|
||||
we emphasize those observations which are misclassified by weighting
|
||||
them with a factor.
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> clf <span style="color: #AA22FF; font-weight: bold">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000">print</span>(clf<span style="color: #666666">.</span><span style="color: #19177C">__class__</span><span style="color: #666666">.</span><span style="color: #19177C">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>log_clf <span style="color: #666666">=</span> LogisticRegression(random_state<span style="color: #666666">=42</span>)
|
||||
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(random_state<span style="color: #666666">=42</span>)
|
||||
svm_clf <span style="color: #666666">=</span> SVC(probability<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, random_state<span style="color: #666666">=42</span>)
|
||||
|
||||
voting_clf <span style="color: #666666">=</span> VotingClassifier(
|
||||
estimators<span style="color: #666666">=</span>[(<span style="color: #BA2121">'lr'</span>, log_clf), (<span style="color: #BA2121">'rf'</span>, rnd_clf), (<span style="color: #BA2121">'svc'</span>, svm_clf)],
|
||||
voting<span style="color: #666666">=</span><span style="color: #BA2121">'soft'</span>)
|
||||
voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> clf <span style="color: #AA22FF; font-weight: bold">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000">print</span>(clf<span style="color: #666666">.</span><span style="color: #19177C">__class__</span><span style="color: #666666">.</span><span style="color: #19177C">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -217,7 +272,7 @@ them with a factor.
|
||||
<li><a href="._week45-bs015.html">16</a></li>
|
||||
<li><a href="._week45-bs016.html">17</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs008.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,54 +193,46 @@ MathJax.Hub.Config({
|
||||
<a name="part0008"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec7" class="anchor">What is boosting? Additive Modelling/Iterative Fitting </h2>
|
||||
<h2 id="___sec7" class="anchor">Random forests </h2>
|
||||
|
||||
<p>
|
||||
Boosting is a way of fitting an additive expansion in a set of
|
||||
elementary basis functions like for example some simple polynomials.
|
||||
Assume for example that we have a function
|
||||
Random forests provide an improvement over bagged trees by way of a
|
||||
small tweak that decorrelates the trees.
|
||||
|
||||
<p>
|
||||
As in bagging, we build a
|
||||
number of decision trees on bootstrapped training samples. But when
|
||||
building these decision trees, each time a split in a tree is
|
||||
considered, a random sample of \( m \) predictors is chosen as split
|
||||
candidates from the full set of \( p \) predictors. The split is allowed to
|
||||
use only one of those \( m \) predictors.
|
||||
|
||||
<p>
|
||||
A fresh sample of \( m \) predictors is
|
||||
taken at each split, and typically we choose
|
||||
|
||||
$$
|
||||
f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m),
|
||||
m\approx \sqrt{p}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
where \( \beta_m \) are the expansion parameters to be determined in a
|
||||
minimization process and \( b(x;\gamma_m) \) are some simple functions of
|
||||
the multivariable parameter \( x \) which is characterized by the
|
||||
parameters \( \gamma_m \).
|
||||
In building a random forest, at
|
||||
each split in the tree, the algorithm is not even allowed to consider
|
||||
a majority of the available predictors.
|
||||
|
||||
<p>
|
||||
As an example, consider the Sigmoid function we used in logistic
|
||||
regression. In that case, we can translate the function
|
||||
\( b(x;\gamma_m) \) into the Sigmoid function
|
||||
|
||||
$$
|
||||
\sigma(t) = \frac{1}{1+\exp{(-t)}},
|
||||
$$
|
||||
|
||||
<p>
|
||||
where \( t=\gamma_0+\gamma_1 x \) and the parameters \( \gamma_0 \) and
|
||||
\( \gamma_1 \) were determined by the Logistic Regression fitting
|
||||
algorithm.
|
||||
|
||||
<p>
|
||||
As another example, consider the cost function we defined for linear regression
|
||||
$$
|
||||
C(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f(x_i))^2.
|
||||
$$
|
||||
|
||||
<p>
|
||||
In this case the function \( f(x) \) was replaced by the design matrix
|
||||
\( \boldsymbol{X} \) and the unknown linear regression parameters \( \boldsymbol{\beta} \),
|
||||
that is \( \boldsymbol{f}=\boldsymbol{X}\boldsymbol{\beta} \). In linear regression we can
|
||||
simply invert a matrix and obtain the parameters \( \beta \) by
|
||||
|
||||
$$
|
||||
\boldsymbol{\beta}=\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
In iterative fitting or additive modeling, we minimize the cost function with respect to the parameters \( \beta_m \) and \( \gamma_m \).
|
||||
The reason for this is rather clever. Suppose that there is one very
|
||||
strong predictor in the data set, along with a number of other
|
||||
moderately strong predictors. Then in the collection of bagged
|
||||
variable importance random forest trees, most or all of the trees will
|
||||
use this strong predictor in the top split. Consequently, all of the
|
||||
bagged trees will look quite similar to each other. Hence the
|
||||
predictions from the bagged trees will be highly correlated.
|
||||
Unfortunately, averaging many highly correlated quantities does not
|
||||
lead to as large of a reduction in variance as averaging many
|
||||
uncorrelated quantities. In particular, this means that bagging will
|
||||
not lead to a substantial reduction in variance over a single tree in
|
||||
this setting.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -253,7 +258,7 @@ In iterative fitting or additive modeling, we minimize the cost function with re
|
||||
<li><a href="._week45-bs016.html">17</a></li>
|
||||
<li><a href="._week45-bs017.html">18</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs009.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,29 +193,30 @@ MathJax.Hub.Config({
|
||||
<a name="part0009"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec8" class="anchor">Iterative Fitting, Regression and Squared-error Cost Function </h2>
|
||||
<h2 id="___sec8" class="anchor">Random Forest Algorithm </h2>
|
||||
The algorithm described here can be applied to both classification and regression problems.
|
||||
|
||||
<p>
|
||||
The way we proceed is as follows (here we specialize to the squared-error cost function)
|
||||
We will grow of forest of say \( B \) trees.
|
||||
|
||||
<ol>
|
||||
<li> Establish a cost function, here \( {\cal C}(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f_M(x_i))^2 \) with \( f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m) \).</li>
|
||||
<li> Initialize with a guess \( f_0(x) \). It could be one or even zero or some random numbers.</li>
|
||||
<li> For \( m=1:M \)
|
||||
<li> For \( b=1:B \)</li>
|
||||
|
||||
<ol type="a"></li>
|
||||
<li> minimize \( \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2 \) wrt \( \gamma \) and \( \beta \)</li>
|
||||
<li> This gives the optimal values \( \beta_m \) and \( \gamma_m \)</li>
|
||||
<li> Determine then the new values \( f_m(x)=f_{m-1}(x) +\beta_m b(x;\gamma_m) \)</li>
|
||||
<ul>
|
||||
<li> Draw a bootstrap sample of from the training data organized in our \( \boldsymbol{X} \) matrix.</li>
|
||||
<li> We grow then a random forest tree \( T_b \) based on the bootstrapped data by repeating the steps outlined till we reach the maximum node size is reached</li>
|
||||
|
||||
<ol>
|
||||
<li> we select \( m \le p \) variables at random from the \( p \) predictors/features</li>
|
||||
<li> pick the best split point among the \( m \) features using either the CART algorithm or the ID3 for classification and create a new node</li>
|
||||
<li> split the node into daughter nodes</li>
|
||||
</ol>
|
||||
|
||||
</ul>
|
||||
|
||||
<li> Output then the ensemble of trees \( \{T_b\}_1^{B} \) and make predictions for either a regression type of problem or a classification type of problem.</li>
|
||||
</ol>
|
||||
|
||||
We could use any of the algorithms we have discussed till now. If we
|
||||
use trees, \( \gamma \) parameterizes the split variables and split points
|
||||
at the internal nodes, and the predictions at the terminal nodes.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
@@ -227,7 +241,7 @@ at the internal nodes, and the predictions at the terminal nodes.
|
||||
<li><a href="._week45-bs017.html">18</a></li>
|
||||
<li><a href="._week45-bs018.html">19</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs010.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,51 +193,76 @@ MathJax.Hub.Config({
|
||||
<a name="part0010"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec9" class="anchor">Squared-Error Example and Iterative Fitting </h2>
|
||||
|
||||
<h2 id="___sec9" class="anchor">Random Forests Compared with other Methods on the Cancer Data </h2>
|
||||
<p>
|
||||
To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function.
|
||||
|
||||
<p>
|
||||
For simplicity we assume also that our functions \( b(x;\gamma)=1+\gamma x \).
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
|
||||
<p>
|
||||
This means that for every iteration \( m \), we need to optimize
|
||||
<span style="color: #408080; font-style: italic"># Load the data</span>
|
||||
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
||||
|
||||
$$
|
||||
(\beta_m,\gamma_m) = \mathrm{argmin}_{\beta,\lambda}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2=\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta(1+\gamma x_i))^2.
|
||||
$$
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
||||
<span style="color: #008000">print</span>(X_train<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #008000">print</span>(X_test<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
||||
logreg <span style="color: #666666">=</span> LogisticRegression(solver<span style="color: #666666">=</span><span style="color: #BA2121">'lbfgs'</span>)
|
||||
logreg<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Logistic Regression: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Support vector machine</span>
|
||||
svm <span style="color: #666666">=</span> SVC(gamma<span style="color: #666666">=</span><span style="color: #BA2121">'auto'</span>, C<span style="color: #666666">=100</span>)
|
||||
svm<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with SVM: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(svm<span style="color: #666666">.</span>score(X_test,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Decision Trees</span>
|
||||
deep_tree_clf <span style="color: #666666">=</span> DecisionTreeClassifier(max_depth<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>)
|
||||
deep_tree_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Decision Trees: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(deep_tree_clf<span style="color: #666666">.</span>score(X_test,y_test)))
|
||||
<span style="color: #408080; font-style: italic">#now scale the data</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
scaler<span style="color: #666666">.</span>fit(X_train)
|
||||
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
||||
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
||||
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
||||
logreg<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy Logistic Regression with scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Support Vector Machine</span>
|
||||
svm<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy SVM with scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Decision Trees</span>
|
||||
deep_tree_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Decision Trees and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(deep_tree_clf<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
|
||||
<p>
|
||||
We start our iteration by simply setting \( f_0(x)=0 \).
|
||||
Taking the derivatives with respect to \( \beta \) and \( \gamma \) we obtain
|
||||
$$
|
||||
\frac{\partial {\cal C}}{\partial \beta} = -2\sum_{i}(1+\gamma x_i)(y_i-\beta(1+\gamma x_i))=0,
|
||||
$$
|
||||
|
||||
and
|
||||
$$
|
||||
\frac{\partial {\cal C}}{\partial \gamma} =-2\sum_{i}\beta x_i(y_i-\beta(1+\gamma x_i))=0.
|
||||
$$
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> LabelEncoder
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
|
||||
<span style="color: #408080; font-style: italic"># Data set not specificied</span>
|
||||
<span style="color: #408080; font-style: italic">#Instantiate the model with 500 trees and entropy as splitting criteria</span>
|
||||
Random_Forest_model <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=500</span>,criterion<span style="color: #666666">=</span><span style="color: #BA2121">"entropy"</span>)
|
||||
Random_Forest_model<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #408080; font-style: italic">#Cross validation</span>
|
||||
accuracy <span style="color: #666666">=</span> cross_validate(Random_Forest_model,X_test_scaled,y_test,cv<span style="color: #666666">=10</span>)[<span style="color: #BA2121">'test_score'</span>]
|
||||
<span style="color: #008000">print</span>(accuracy)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Random Forests and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(Random_Forest_model<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
|
||||
We can then rewrite these equations as (defining \( \boldsymbol{w}=\boldsymbol{e}+\gamma \boldsymbol{x}) \) with \( \boldsymbol{e} \) being the unit vector)
|
||||
$$
|
||||
\gamma \boldsymbol{w}^T(\boldsymbol{y}-\beta\gamma \boldsymbol{w})=0,
|
||||
$$
|
||||
|
||||
which gives us \( \beta = \boldsymbol{w}^T\boldsymbol{y}/(\boldsymbol{w}^T\boldsymbol{w}) \). Similarly we have
|
||||
$$
|
||||
\beta\gamma \boldsymbol{x}^T(\boldsymbol{y}-\beta(1+\gamma \boldsymbol{x}))=0,
|
||||
$$
|
||||
|
||||
<p>
|
||||
which leads to \( \gamma =(\boldsymbol{x}^T\boldsymbol{y}-\beta\boldsymbol{x}^T\boldsymbol{e})/(\beta\boldsymbol{x}^T\boldsymbol{x}) \). Inserting
|
||||
for \( \beta \) gives us an equation for \( \gamma \). This is a non-linear equation in the unknown \( \gamma \) and has to be solved numerically.
|
||||
|
||||
<p>
|
||||
The solution to these two equations gives us in turn \( \beta_1 \) and \( \gamma_1 \) leading to the new expression for \( f_1(x) \) as
|
||||
\( f_1(x) = \beta_1(1+\gamma_1x) \). Doing this \( M \) times results in our final estimate for the function \( f \).
|
||||
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
y_pred <span style="color: #666666">=</span> Random_Forest_model<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
y_probas <span style="color: #666666">=</span> Random_Forest_model<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -251,7 +289,7 @@ The solution to these two equations gives us in turn \( \beta_1 \) and \( \gamma
|
||||
<li><a href="._week45-bs018.html">19</a></li>
|
||||
<li><a href="._week45-bs019.html">20</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs011.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,38 +193,25 @@ MathJax.Hub.Config({
|
||||
<a name="part0011"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec10" class="anchor">Iterative Fitting, Classification and AdaBoost </h2>
|
||||
|
||||
<h2 id="___sec10" class="anchor">Compare Bagging on Trees with Random Forests </h2>
|
||||
<p>
|
||||
Let us consider a binary classification problem with two outcomes \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
|
||||
observations. We define a classification function \( G(x) \) which produces a prediction taking one or the other of the two values
|
||||
\( \{-1,1\} \).
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>bag_clf <span style="color: #666666">=</span> BaggingClassifier(
|
||||
DecisionTreeClassifier(splitter<span style="color: #666666">=</span><span style="color: #BA2121">"random"</span>, max_leaf_nodes<span style="color: #666666">=16</span>, random_state<span style="color: #666666">=42</span>),
|
||||
n_estimators<span style="color: #666666">=500</span>, max_samples<span style="color: #666666">=1.0</span>, bootstrap<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, n_jobs<span style="color: #666666">=-1</span>, random_state<span style="color: #666666">=42</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
The error rate of the training sample is then
|
||||
|
||||
$$
|
||||
\mathrm{\overline{err}}=\frac{1}{n} \sum_{i=0}^{n-1} I(y_i\ne G(x_i)).
|
||||
$$
|
||||
|
||||
<p>
|
||||
The iterative procedure starts with defining a weak classifier whose
|
||||
error rate is barely better than random guessing. The iterative
|
||||
procedure in boosting is to sequentially apply a weak
|
||||
classification algorithm to repeatedly modified versions of the data
|
||||
producing a sequence of weak classifiers \( G_m(x) \).
|
||||
|
||||
<p>
|
||||
Here we will express our function \( f(x) \) in terms of \( G(x) \). That is
|
||||
$$
|
||||
f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m),
|
||||
$$
|
||||
|
||||
will be a function of
|
||||
$$
|
||||
G_M(x) = \mathrm{sign} \sum_{i=1}^M \alpha_m G_m(x).
|
||||
$$
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>bag_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred <span style="color: #666666">=</span> bag_clf<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
|
||||
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=500</span>, max_leaf_nodes<span style="color: #666666">=16</span>, n_jobs<span style="color: #666666">=-1</span>, random_state<span style="color: #666666">=42</span>)
|
||||
rnd_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred_rf <span style="color: #666666">=</span> rnd_clf<span style="color: #666666">.</span>predict(X_test)
|
||||
np<span style="color: #666666">.</span>sum(y_pred <span style="color: #666666">==</span> y_pred_rf) <span style="color: #666666">/</span> <span style="color: #008000">len</span>(y_pred)
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -238,7 +238,7 @@ $$
|
||||
<li><a href="._week45-bs019.html">20</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs012.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,30 +193,19 @@ MathJax.Hub.Config({
|
||||
<a name="part0012"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec11" class="anchor">Adaptive Boosting, AdaBoost </h2>
|
||||
<h2 id="___sec11" class="anchor">Boosting, a Bird's Eye View </h2>
|
||||
|
||||
<p>
|
||||
In our iterative procedure we define thus
|
||||
$$
|
||||
f_m(x) = f_{m-1}(x)+\beta_mG_m(x).
|
||||
$$
|
||||
The basic idea is to combine weak classifiers in order to create a good
|
||||
classifier. With a weak classifier we often intend a classifier which
|
||||
produces results which are only slightly better than we would get by
|
||||
random guesses.
|
||||
|
||||
<p>
|
||||
The simplest possible cost function which leads (also simple from a computational point of view) to the AdaBoost algorithm is the
|
||||
exponential cost/loss function defined as
|
||||
$$
|
||||
C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}\exp{(-y_i(f_{m-1}(x_i)+\beta G(x_i))}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
We optimize \( \beta \) and \( G \) for each value of \( m=1:M \) as we did in the regression case.
|
||||
This is normally done in two steps. Let us however first rewrite the cost function as
|
||||
|
||||
$$
|
||||
C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}w_i^{m}\exp{(-y_i\beta G(x_i))},
|
||||
$$
|
||||
|
||||
where we have defined \( w_i^m= \exp{(-y_if_{m-1}(x_i))} \).
|
||||
This is done by applying in an iterative way a weak (or a standard
|
||||
classifier like decision trees) to modify the data. In each iteration
|
||||
we emphasize those observations which are misclassified by weighting
|
||||
them with a factor.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -231,7 +233,7 @@ where we have defined \( w_i^m= \exp{(-y_if_{m-1}(x_i))} \).
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs013.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,46 +193,54 @@ MathJax.Hub.Config({
|
||||
<a name="part0013"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec12" class="anchor">Building up AdaBoost </h2>
|
||||
<h2 id="___sec12" class="anchor">What is boosting? Additive Modelling/Iterative Fitting </h2>
|
||||
|
||||
<p>
|
||||
First, for any \( \beta > 0 \), we optimize \( G \) by setting
|
||||
Boosting is a way of fitting an additive expansion in a set of
|
||||
elementary basis functions like for example some simple polynomials.
|
||||
Assume for example that we have a function
|
||||
$$
|
||||
G_m(x) = \mathrm{sign} \sum_{i=0}^{n-1} w_i^m I(y_i \ne G_(x_i)),
|
||||
f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m),
|
||||
$$
|
||||
|
||||
which is the classifier that minimizes the weighted error rate in predicting \( y \).
|
||||
|
||||
<p>
|
||||
We can do this by rewriting
|
||||
where \( \beta_m \) are the expansion parameters to be determined in a
|
||||
minimization process and \( b(x;\gamma_m) \) are some simple functions of
|
||||
the multivariable parameter \( x \) which is characterized by the
|
||||
parameters \( \gamma_m \).
|
||||
|
||||
<p>
|
||||
As an example, consider the Sigmoid function we used in logistic
|
||||
regression. In that case, we can translate the function
|
||||
\( b(x;\gamma_m) \) into the Sigmoid function
|
||||
|
||||
$$
|
||||
\exp{-(\beta)}\sum_{y_i=G(x_i)}w_i^m+\exp{(\beta)}\sum_{y_i\ne G(x_i)}w_i^m,
|
||||
\sigma(t) = \frac{1}{1+\exp{(-t)}},
|
||||
$$
|
||||
|
||||
which can be rewritten as
|
||||
<p>
|
||||
where \( t=\gamma_0+\gamma_1 x \) and the parameters \( \gamma_0 \) and
|
||||
\( \gamma_1 \) were determined by the Logistic Regression fitting
|
||||
algorithm.
|
||||
|
||||
<p>
|
||||
As another example, consider the cost function we defined for linear regression
|
||||
$$
|
||||
(\exp{(\beta)}-\exp{-(\beta)})\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i))+\exp{(-\beta)}\sum_{i=0}^{n-1}w_i^m=0,
|
||||
C(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f(x_i))^2.
|
||||
$$
|
||||
|
||||
which leads to
|
||||
<p>
|
||||
In this case the function \( f(x) \) was replaced by the design matrix
|
||||
\( \boldsymbol{X} \) and the unknown linear regression parameters \( \boldsymbol{\beta} \),
|
||||
that is \( \boldsymbol{f}=\boldsymbol{X}\boldsymbol{\beta} \). In linear regression we can
|
||||
simply invert a matrix and obtain the parameters \( \beta \) by
|
||||
|
||||
$$
|
||||
\beta_m = \frac{1}{2}\log{\frac{1-\mathrm{\overline{err}}}{\mathrm{\overline{err}}}},
|
||||
\boldsymbol{\beta}=\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
|
||||
$$
|
||||
|
||||
where we have redefined the error as
|
||||
$$
|
||||
\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i)}{\sum_{i=0}^{n-1}w_i^m},
|
||||
$$
|
||||
|
||||
which leads to an update of
|
||||
$$
|
||||
f_m(x) = f_{m-1}(x) +\beta_m G_m(x).
|
||||
$$
|
||||
|
||||
This leads to the new weights
|
||||
$$
|
||||
w_i^{m+1} = w_i^m \exp{(-y_i\beta_m G_m(x_i))}
|
||||
$$
|
||||
<p>
|
||||
In iterative fitting or additive modeling, we minimize the cost function with respect to the parameters \( \beta_m \) and \( \gamma_m \).
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -247,7 +268,7 @@ $$
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs022.html">23</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs014.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,24 +193,27 @@ MathJax.Hub.Config({
|
||||
<a name="part0014"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec13" class="anchor">Adaptive boosting: AdaBoost, Basic Algorithm </h2>
|
||||
<h2 id="___sec13" class="anchor">Iterative Fitting, Regression and Squared-error Cost Function </h2>
|
||||
|
||||
<p>
|
||||
The algorithm here is rather straightforward. Assume that our weak
|
||||
classifier is a decision tree and we consider a binary set of outputs
|
||||
with \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
|
||||
observations. Our design matrix is given in terms of the
|
||||
feature/predictor vectors
|
||||
\( \boldsymbol{X}=[\boldsymbol{x}_0\boldsymbol{x}_1\dots\boldsymbol{x}_{p-1}] \). Finally, we define also a
|
||||
classifier determined by our data via a function \( G(x) \). This function tells us how well we are able to classify our outputs/targets \( \boldsymbol{y} \).
|
||||
The way we proceed is as follows (here we specialize to the squared-error cost function)
|
||||
|
||||
<p>
|
||||
We have already defined the misclassification error \( \mathrm{err} \) as
|
||||
$$
|
||||
\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(x_i)),
|
||||
$$
|
||||
<ol>
|
||||
<li> Establish a cost function, here \( {\cal C}(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f_M(x_i))^2 \) with \( f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m) \).</li>
|
||||
<li> Initialize with a guess \( f_0(x) \). It could be one or even zero or some random numbers.</li>
|
||||
<li> For \( m=1:M \)
|
||||
|
||||
where the function \( I() \) is one if we misclassify and zero if we classify correctly.
|
||||
<ol type="a"></li>
|
||||
<li> minimize \( \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2 \) wrt \( \gamma \) and \( \beta \)</li>
|
||||
<li> This gives the optimal values \( \beta_m \) and \( \gamma_m \)</li>
|
||||
<li> Determine then the new values \( f_m(x)=f_{m-1}(x) +\beta_m b(x;\gamma_m) \)</li>
|
||||
</ol>
|
||||
|
||||
</ol>
|
||||
|
||||
We could use any of the algorithms we have discussed till now. If we
|
||||
use trees, \( \gamma \) parameterizes the split variables and split points
|
||||
at the internal nodes, and the predictions at the terminal nodes.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -225,7 +241,7 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
|
||||
<li><a href="._week45-bs022.html">23</a></li>
|
||||
<li><a href="._week45-bs023.html">24</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs015.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,42 +193,50 @@ MathJax.Hub.Config({
|
||||
<a name="part0015"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec14" class="anchor">Basic Steps of AdaBoost </h2>
|
||||
<h2 id="___sec14" class="anchor">Squared-Error Example and Iterative Fitting </h2>
|
||||
|
||||
<p>
|
||||
With the above definitions we are now ready to set up the algorithm for AdaBoost.
|
||||
The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases.
|
||||
To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function.
|
||||
|
||||
<ol>
|
||||
<li> We start by initializing all weights to \( w_i = 1/n \), with \( i=0,1,2,\dots n-1 \). It is easy to see that we must have \( \sum_{i=0}^{n-1}w_i = 1 \).</li>
|
||||
<li> We rewrite the misclassification error as</li>
|
||||
</ol>
|
||||
<p>
|
||||
For simplicity we assume also that our functions \( b(x;\gamma)=1+\gamma x \).
|
||||
|
||||
<p>
|
||||
This means that for every iteration \( m \), we need to optimize
|
||||
|
||||
$$
|
||||
\mathrm{\overline{err}}_m=\frac{\sum_{i=0}^{n-1}w_i^m I(y_i\ne G(x_i))}{\sum_{i=0}^{n-1}w_i},
|
||||
(\beta_m,\gamma_m) = \mathrm{argmin}_{\beta,\lambda}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2=\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta(1+\gamma x_i))^2.
|
||||
$$
|
||||
|
||||
<p>
|
||||
We start our iteration by simply setting \( f_0(x)=0 \).
|
||||
Taking the derivatives with respect to \( \beta \) and \( \gamma \) we obtain
|
||||
$$
|
||||
\frac{\partial {\cal C}}{\partial \beta} = -2\sum_{i}(1+\gamma x_i)(y_i-\beta(1+\gamma x_i))=0,
|
||||
$$
|
||||
|
||||
<ol>
|
||||
<li> Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree.
|
||||
and
|
||||
$$
|
||||
\frac{\partial {\cal C}}{\partial \gamma} =-2\sum_{i}\beta x_i(y_i-\beta(1+\gamma x_i))=0.
|
||||
$$
|
||||
|
||||
<ol type="a"></li>
|
||||
<li> Fit then a given classifier to the training set using the weights \( w_i \).</li>
|
||||
<li> Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.</li>
|
||||
<li> Define a quantity \( \alpha_{m} = \log{(1-\mathrm{\overline{err}}_m)/\mathrm{\overline{err}}_m} \)</li>
|
||||
<li> Set the new weights to \( w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(x_i)} \).</li>
|
||||
</ol>
|
||||
We can then rewrite these equations as (defining \( \boldsymbol{w}=\boldsymbol{e}+\gamma \boldsymbol{x}) \) with \( \boldsymbol{e} \) being the unit vector)
|
||||
$$
|
||||
\gamma \boldsymbol{w}^T(\boldsymbol{y}-\beta\gamma \boldsymbol{w})=0,
|
||||
$$
|
||||
|
||||
<li> Compute the new classifier \( G(x)= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(x_i) \).</li>
|
||||
</ol>
|
||||
which gives us \( \beta = \boldsymbol{w}^T\boldsymbol{y}/(\boldsymbol{w}^T\boldsymbol{w}) \). Similarly we have
|
||||
$$
|
||||
\beta\gamma \boldsymbol{x}^T(\boldsymbol{y}-\beta(1+\gamma \boldsymbol{x}))=0,
|
||||
$$
|
||||
|
||||
For the iterations with \( m \le 2 \) the weights are modified
|
||||
individually at each steps. The observations which were misclassified
|
||||
at iteration \( m-1 \) have a weight which is larger than those which were
|
||||
classified properly. As this proceeds, the observations which were
|
||||
difficult to classifiy correctly are given a larger influence. Each
|
||||
new classification step \( m \) is then forced to concentrate on those
|
||||
observations that are missed in the previous iterations.
|
||||
<p>
|
||||
which leads to \( \gamma =(\boldsymbol{x}^T\boldsymbol{y}-\beta\boldsymbol{x}^T\boldsymbol{e})/(\beta\boldsymbol{x}^T\boldsymbol{x}) \). Inserting
|
||||
for \( \beta \) gives us an equation for \( \gamma \). This is a non-linear equation in the unknown \( \gamma \) and has to be solved numerically.
|
||||
|
||||
<p>
|
||||
The solution to these two equations gives us in turn \( \beta_1 \) and \( \gamma_1 \) leading to the new expression for \( f_1(x) \) as
|
||||
\( f_1(x) = \beta_1(1+\gamma_1x) \). Doing this \( M \) times results in our final estimate for the function \( f \).
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -243,7 +264,7 @@ observations that are missed in the previous iterations.
|
||||
<li><a href="._week45-bs023.html">24</a></li>
|
||||
<li><a href="._week45-bs024.html">25</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs016.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,36 +193,38 @@ MathJax.Hub.Config({
|
||||
<a name="part0016"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec15" class="anchor">AdaBoost Examples </h2>
|
||||
<h2 id="___sec15" class="anchor">Iterative Fitting, Classification and AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
Using <b>Scikit-Learn</b> it is easy to apply the adaptive boosting algorithm, as done here.
|
||||
Let us consider a binary classification problem with two outcomes \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
|
||||
observations. We define a classification function \( G(x) \) which produces a prediction taking one or the other of the two values
|
||||
\( \{-1,1\} \).
|
||||
|
||||
<p>
|
||||
The error rate of the training sample is then
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> AdaBoostClassifier
|
||||
$$
|
||||
\mathrm{\overline{err}}=\frac{1}{n} \sum_{i=0}^{n-1} I(y_i\ne G(x_i)).
|
||||
$$
|
||||
|
||||
ada_clf <span style="color: #666666">=</span> AdaBoostClassifier(
|
||||
DecisionTreeClassifier(max_depth<span style="color: #666666">=1</span>), n_estimators<span style="color: #666666">=200</span>,
|
||||
algorithm<span style="color: #666666">=</span><span style="color: #BA2121">"SAMME.R"</span>, learning_rate<span style="color: #666666">=0.5</span>, random_state<span style="color: #666666">=42</span>)
|
||||
ada_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
<p>
|
||||
The iterative procedure starts with defining a weak classifier whose
|
||||
error rate is barely better than random guessing. The iterative
|
||||
procedure in boosting is to sequentially apply a weak
|
||||
classification algorithm to repeatedly modified versions of the data
|
||||
producing a sequence of weak classifiers \( G_m(x) \).
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> AdaBoostClassifier
|
||||
<p>
|
||||
Here we will express our function \( f(x) \) in terms of \( G(x) \). That is
|
||||
$$
|
||||
f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m),
|
||||
$$
|
||||
|
||||
will be a function of
|
||||
$$
|
||||
G_M(x) = \mathrm{sign} \sum_{i=1}^M \alpha_m G_m(x).
|
||||
$$
|
||||
|
||||
ada_clf <span style="color: #666666">=</span> AdaBoostClassifier(
|
||||
DecisionTreeClassifier(max_depth<span style="color: #666666">=1</span>), n_estimators<span style="color: #666666">=200</span>,
|
||||
algorithm<span style="color: #666666">=</span><span style="color: #BA2121">"SAMME.R"</span>, learning_rate<span style="color: #666666">=0.5</span>, random_state<span style="color: #666666">=42</span>)
|
||||
ada_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
y_pred <span style="color: #666666">=</span> ada_clf<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
y_probas <span style="color: #666666">=</span> ada_clf<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -236,7 +251,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._week45-bs024.html">25</a></li>
|
||||
<li><a href="._week45-bs025.html">26</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs017.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,30 +193,30 @@ MathJax.Hub.Config({
|
||||
<a name="part0017"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec16" class="anchor">AdaBoost for Regression </h2>
|
||||
<h2 id="___sec16" class="anchor">Adaptive Boosting, AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
Here we present <a href="https://pdfs.semanticscholar.org/8d49/e2dedb817f2c3330e74b63c5fc86d2399ce3.pdf" target="_self">Drucker's AdaBoost</a> tailored for regression.
|
||||
In our iterative procedure we define thus
|
||||
$$
|
||||
f_m(x) = f_{m-1}(x)+\beta_mG_m(x).
|
||||
$$
|
||||
|
||||
<p>
|
||||
In bagging, each training example is equally likely to be
|
||||
picked. In boosting, the probability of a particular
|
||||
example being in the training set of a particular machine
|
||||
depends on the performance of the prior machines on
|
||||
that example. The following is a modification of
|
||||
Adaboost by Drucker.
|
||||
The simplest possible cost function which leads (also simple from a computational point of view) to the AdaBoost algorithm is the
|
||||
exponential cost/loss function defined as
|
||||
$$
|
||||
C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}\exp{(-y_i(f_{m-1}(x_i)+\beta G(x_i))}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
Start by selecting a set of training data \( n \) and assign to each entry a weight \( w_i=1 \) for \( i=1,2,\dots,n \). As we have done earlier, we could pick say \( 80\% \) of the data set for training. The algorithm runs as follows:
|
||||
We optimize \( \beta \) and \( G \) for each value of \( m=1:M \) as we did in the regression case.
|
||||
This is normally done in two steps. Let us however first rewrite the cost function as
|
||||
|
||||
<ol>
|
||||
<li> We define the probability that the training sample \( i \) is in the set by \( p_i = w_i/\sum_iw_i \). We pick \( n \) samples (with replacement) to form our training set. We pick a number uniformly in the range \( [0,\sum_iw_i] \).</li>
|
||||
<li> We choose then a regression machine (for example plain linear regression or a simple decision tree). A given regression machine makes then a hypothesis.</li>
|
||||
<li> Using every member of the training set with the chosen regression machine we obtain then a prediction \( \tilde{y}_i \).</li>
|
||||
<li> We calculate then the loss function \( L_i \) for each training sample. We can use various types of loss function as long as we have a value</li>
|
||||
</ol>
|
||||
$$
|
||||
C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}w_i^{m}\exp{(-y_i\beta G(x_i))},
|
||||
$$
|
||||
|
||||
\( L_i\in [0,1] \).
|
||||
where we have defined \( w_i^m= \exp{(-y_if_{m-1}(x_i))} \).
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -231,7 +244,7 @@ Start by selecting a set of training data \( n \) and assign to each entry a wei
|
||||
<li><a href="._week45-bs025.html">26</a></li>
|
||||
<li><a href="._week45-bs026.html">27</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs018.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,17 +193,46 @@ MathJax.Hub.Config({
|
||||
<a name="part0018"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec17" class="anchor">Gradient boosting: Basics with Steepest Descent </h2>
|
||||
<h2 id="___sec17" class="anchor">Building up AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
Gradient boosting is again a similar technique to Adaptive boosting,
|
||||
it combines so-called weak classifiers or regressors into a strong
|
||||
method via a series of iterations.
|
||||
First, for any \( \beta > 0 \), we optimize \( G \) by setting
|
||||
$$
|
||||
G_m(x) = \mathrm{sign} \sum_{i=0}^{n-1} w_i^m I(y_i \ne G_(x_i)),
|
||||
$$
|
||||
|
||||
which is the classifier that minimizes the weighted error rate in predicting \( y \).
|
||||
|
||||
<p>
|
||||
In order to understand the method, let us illustrate its basics by
|
||||
bringing back the essential steps in linear regression, where our cost
|
||||
function was the least squares function.
|
||||
We can do this by rewriting
|
||||
$$
|
||||
\exp{-(\beta)}\sum_{y_i=G(x_i)}w_i^m+\exp{(\beta)}\sum_{y_i\ne G(x_i)}w_i^m,
|
||||
$$
|
||||
|
||||
which can be rewritten as
|
||||
$$
|
||||
(\exp{(\beta)}-\exp{-(\beta)})\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i))+\exp{(-\beta)}\sum_{i=0}^{n-1}w_i^m=0,
|
||||
$$
|
||||
|
||||
which leads to
|
||||
$$
|
||||
\beta_m = \frac{1}{2}\log{\frac{1-\mathrm{\overline{err}}}{\mathrm{\overline{err}}}},
|
||||
$$
|
||||
|
||||
where we have redefined the error as
|
||||
$$
|
||||
\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i)}{\sum_{i=0}^{n-1}w_i^m},
|
||||
$$
|
||||
|
||||
which leads to an update of
|
||||
$$
|
||||
f_m(x) = f_{m-1}(x) +\beta_m G_m(x).
|
||||
$$
|
||||
|
||||
This leads to the new weights
|
||||
$$
|
||||
w_i^{m+1} = w_i^m \exp{(-y_i\beta_m G_m(x_i))}
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -217,6 +259,8 @@ function was the least squares function.
|
||||
<li><a href="._week45-bs025.html">26</a></li>
|
||||
<li><a href="._week45-bs026.html">27</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs019.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,37 +193,24 @@ MathJax.Hub.Config({
|
||||
<a name="part0019"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec18" class="anchor">The Squared-Error again! Steepest Descent </h2>
|
||||
<h2 id="___sec18" class="anchor">Adaptive boosting: AdaBoost, Basic Algorithm </h2>
|
||||
|
||||
<p>
|
||||
We start again with our cost function \( {\cal C}(\boldsymbol{y}m\boldsymbol{f})=\sum_{i=0}^{n-1}{\cal L}(y_i, f(x_i)) \) where we want to minimize
|
||||
This means that for every iteration, we need to optimize
|
||||
|
||||
$$
|
||||
(\hat{\boldsymbol{f}}) = \mathrm{argmin}_{\boldsymbol{f}}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f(x_i))^2.
|
||||
$$
|
||||
The algorithm here is rather straightforward. Assume that our weak
|
||||
classifier is a decision tree and we consider a binary set of outputs
|
||||
with \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
|
||||
observations. Our design matrix is given in terms of the
|
||||
feature/predictor vectors
|
||||
\( \boldsymbol{X}=[\boldsymbol{x}_0\boldsymbol{x}_1\dots\boldsymbol{x}_{p-1}] \). Finally, we define also a
|
||||
classifier determined by our data via a function \( G(x) \). This function tells us how well we are able to classify our outputs/targets \( \boldsymbol{y} \).
|
||||
|
||||
<p>
|
||||
We define a real function \( h_m(x) \) that defines our final function \( f_M(x) \) as
|
||||
We have already defined the misclassification error \( \mathrm{err} \) as
|
||||
$$
|
||||
f_M(x) = \sum_{m=0}^M h_m(x).
|
||||
\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(x_i)),
|
||||
$$
|
||||
|
||||
<p>
|
||||
In the steepest decent approach we approximate \( h_m(x) = -\rho_m g_m(x) \), where \( \rho_m \) is a scalar and \( g_m(x) \) the gradient defined as
|
||||
$$
|
||||
g_m(x_i) = \left[ \frac{\partial {\cal L}(y_i, f(x_i))}{\partial f(x_i)}\right]_{f(x_i)=f_{m-1}(x_i)}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
With the new gradient we can update \( f_m(x) = f_{m-1}(x) -\rho_m g_m(x) \). Using the above squared-error function we see that
|
||||
the gradient is \( g_m(x_i) = -2(y_i-f(x_i)) \).
|
||||
|
||||
<p>
|
||||
Choosing \( f_0(x)=0 \) we obtain \( g_m(x) = -2y_i \) and inserting this into the minimization problem for the cost function we have
|
||||
$$
|
||||
(\rho_1) = \mathrm{argmin}_{\rho}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i+2\rho y_i)^2.
|
||||
$$
|
||||
where the function \( I() \) is one if we misclassify and zero if we classify correctly.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -236,6 +236,9 @@ $$
|
||||
<li><a href="._week45-bs025.html">26</a></li>
|
||||
<li><a href="._week45-bs026.html">27</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs028.html">29</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs020.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,20 +193,42 @@ MathJax.Hub.Config({
|
||||
<a name="part0020"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec19" class="anchor">Steepest Descent Example </h2>
|
||||
<h2 id="___sec19" class="anchor">Basic Steps of AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
Optimizing with respect to \( \rho \) we obtain (taking the derivative) that \( \rho_1 = -1/2 \). We have then that
|
||||
With the above definitions we are now ready to set up the algorithm for AdaBoost.
|
||||
The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases.
|
||||
|
||||
<ol>
|
||||
<li> We start by initializing all weights to \( w_i = 1/n \), with \( i=0,1,2,\dots n-1 \). It is easy to see that we must have \( \sum_{i=0}^{n-1}w_i = 1 \).</li>
|
||||
<li> We rewrite the misclassification error as</li>
|
||||
</ol>
|
||||
|
||||
$$
|
||||
f_1(x) = f_{0}(x) -\rho_1 g_1(x)=-y_i.
|
||||
\mathrm{\overline{err}}_m=\frac{\sum_{i=0}^{n-1}w_i^m I(y_i\ne G(x_i))}{\sum_{i=0}^{n-1}w_i},
|
||||
$$
|
||||
|
||||
We can then proceed and compute
|
||||
$$
|
||||
g_2(x_i) = \left[ \frac{\partial {\cal L}(y_i, f(x_i))}{\partial f(x_i)}\right]_{f(x_i)=f_{1}(x_i)=y_i}=-4y_i,
|
||||
$$
|
||||
|
||||
and find a new value for \( \rho_2=-1/2 \) and continue till we have reached \( m=M \). We can modify the steepest descent method, or steepest boosting, by introducing what is called <b>gradient boosting</b>.
|
||||
<ol>
|
||||
<li> Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree.
|
||||
|
||||
<ol type="a"></li>
|
||||
<li> Fit then a given classifier to the training set using the weights \( w_i \).</li>
|
||||
<li> Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.</li>
|
||||
<li> Define a quantity \( \alpha_{m} = \log{(1-\mathrm{\overline{err}}_m)/\mathrm{\overline{err}}_m} \)</li>
|
||||
<li> Set the new weights to \( w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(x_i)} \).</li>
|
||||
</ol>
|
||||
|
||||
<li> Compute the new classifier \( G(x)= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(x_i) \).</li>
|
||||
</ol>
|
||||
|
||||
For the iterations with \( m \le 2 \) the weights are modified
|
||||
individually at each steps. The observations which were misclassified
|
||||
at iteration \( m-1 \) have a weight which is larger than those which were
|
||||
classified properly. As this proceeds, the observations which were
|
||||
difficult to classifiy correctly are given a larger influence. Each
|
||||
new classification step \( m \) is then forced to concentrate on those
|
||||
observations that are missed in the previous iterations.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -218,6 +253,10 @@ and find a new value for \( \rho_2=-1/2 \) and continue till we have reached \(
|
||||
<li><a href="._week45-bs025.html">26</a></li>
|
||||
<li><a href="._week45-bs026.html">27</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs028.html">29</a></li>
|
||||
<li><a href="._week45-bs029.html">30</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs021.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,30 +193,37 @@ MathJax.Hub.Config({
|
||||
<a name="part0021"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec20" class="anchor">Gradient Boosting, algorithm </h2>
|
||||
<h2 id="___sec20" class="anchor">AdaBoost Examples </h2>
|
||||
|
||||
<p>
|
||||
Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard squared-error function
|
||||
$$
|
||||
C(\boldsymbol{y},\boldsymbol{f})=\sum_{i=0}^{n-1}(y_i-f(x_i))^2.
|
||||
$$
|
||||
Using <b>Scikit-Learn</b> it is easy to apply the adaptive boosting algorithm, as done here.
|
||||
|
||||
<p>
|
||||
The way we proceed in an iterative fashion is to
|
||||
|
||||
<ol>
|
||||
<li> Initialize our estimate \( f_0(x) \).</li>
|
||||
<li> For \( m=1:M \), we
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> AdaBoostClassifier
|
||||
|
||||
<ol type="a"></li>
|
||||
<li> compute the negative gradient vector \( \boldsymbol{u}_m = -\partial C(\boldsymbol{y},\boldsymbol{f})/\partial \boldsymbol{f}(x) \) at \( f(x) = f_{m-1}(x) \);</li>
|
||||
<li> fit the so-called base-learner to the negative gradient \( h_m(u_m,x) \);</li>
|
||||
<li> update the estimate \( f_m(x) = f_{m-1}(x)+\nu h_m(u_m,x) \);</li>
|
||||
</ol>
|
||||
ada_clf <span style="color: #666666">=</span> AdaBoostClassifier(
|
||||
DecisionTreeClassifier(max_depth<span style="color: #666666">=1</span>), n_estimators<span style="color: #666666">=200</span>,
|
||||
algorithm<span style="color: #666666">=</span><span style="color: #BA2121">"SAMME.R"</span>, learning_rate<span style="color: #666666">=0.5</span>, random_state<span style="color: #666666">=42</span>)
|
||||
ada_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
|
||||
<li> The final estimate is then \( f_M(x) = \sum_{m=1}^M\nu h_m(u_m,x) \).</li>
|
||||
</ol>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> AdaBoostClassifier
|
||||
|
||||
ada_clf <span style="color: #666666">=</span> AdaBoostClassifier(
|
||||
DecisionTreeClassifier(max_depth<span style="color: #666666">=1</span>), n_estimators<span style="color: #666666">=200</span>,
|
||||
algorithm<span style="color: #666666">=</span><span style="color: #BA2121">"SAMME.R"</span>, learning_rate<span style="color: #666666">=0.5</span>, random_state<span style="color: #666666">=42</span>)
|
||||
ada_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
y_pred <span style="color: #666666">=</span> ada_clf<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
y_probas <span style="color: #666666">=</span> ada_clf<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
@@ -225,6 +245,11 @@ The way we proceed in an iterative fashion is to
|
||||
<li><a href="._week45-bs025.html">26</a></li>
|
||||
<li><a href="._week45-bs026.html">27</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs028.html">29</a></li>
|
||||
<li><a href="._week45-bs029.html">30</a></li>
|
||||
<li><a href="._week45-bs030.html">31</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs022.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,10 +193,30 @@ MathJax.Hub.Config({
|
||||
<a name="part0022"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec21" class="anchor">Gradient Boosting Example, Regression </h2>
|
||||
<h2 id="___sec21" class="anchor">AdaBoost for Regression </h2>
|
||||
|
||||
<p>
|
||||
We discuss here the difference between the steepest descent approach and gradient boosting by repeating our simple regression example above.
|
||||
Here we present <a href="https://pdfs.semanticscholar.org/8d49/e2dedb817f2c3330e74b63c5fc86d2399ce3.pdf" target="_self">Drucker's AdaBoost</a> tailored for regression.
|
||||
|
||||
<p>
|
||||
In bagging, each training example is equally likely to be
|
||||
picked. In boosting, the probability of a particular
|
||||
example being in the training set of a particular machine
|
||||
depends on the performance of the prior machines on
|
||||
that example. The following is a modification of
|
||||
Adaboost by Drucker.
|
||||
|
||||
<p>
|
||||
Start by selecting a set of training data \( n \) and assign to each entry a weight \( w_i=1 \) for \( i=1,2,\dots,n \). As we have done earlier, we could pick say \( 80\% \) of the data set for training. The algorithm runs as follows:
|
||||
|
||||
<ol>
|
||||
<li> We define the probability that the training sample \( i \) is in the set by \( p_i = w_i/\sum_iw_i \). We pick \( n \) samples (with replacement) to form our training set. We pick a number uniformly in the range \( [0,\sum_iw_i] \).</li>
|
||||
<li> We choose then a regression machine (for example plain linear regression or a simple decision tree). A given regression machine makes then a hypothesis.</li>
|
||||
<li> Using every member of the training set with the chosen regression machine we obtain then a prediction \( \tilde{y}_i \).</li>
|
||||
<li> We calculate then the loss function \( L_i \) for each training sample. We can use various types of loss function as long as we have a value</li>
|
||||
</ol>
|
||||
|
||||
\( L_i\in [0,1] \).
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -206,6 +239,12 @@ We discuss here the difference between the steepest descent approach and gradien
|
||||
<li><a href="._week45-bs025.html">26</a></li>
|
||||
<li><a href="._week45-bs026.html">27</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs028.html">29</a></li>
|
||||
<li><a href="._week45-bs029.html">30</a></li>
|
||||
<li><a href="._week45-bs030.html">31</a></li>
|
||||
<li><a href="._week45-bs031.html">32</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs023.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,57 +193,18 @@ MathJax.Hub.Config({
|
||||
<a name="part0023"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec22" class="anchor">Gradient Boosting, Examples of Regression </h2>
|
||||
<h2 id="___sec22" class="anchor">Gradient boosting: Basics with Steepest Descent </h2>
|
||||
|
||||
<p>
|
||||
Gradient boosting is again a similar technique to Adaptive boosting,
|
||||
it combines so-called weak classifiers or regressors into a strong
|
||||
method via a series of iterations.
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> GradientBoostingRegressor
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> mean_squared_error
|
||||
<p>
|
||||
In order to understand the method, let us illustrate its basics by
|
||||
bringing back the essential steps in linear regression, where our cost
|
||||
function was the least squares function.
|
||||
|
||||
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
maxdegree <span style="color: #666666">=</span> <span style="color: #666666">6</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Make data set.</span>
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">-3</span>, <span style="color: #666666">3</span>, n)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x<span style="color: #666666">**2</span>) <span style="color: #666666">+</span> <span style="color: #666666">1.5</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(x<span style="color: #666666">-2</span>)<span style="color: #666666">**2</span>)<span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal(<span style="color: #666666">0</span>, <span style="color: #666666">0.1</span>, x<span style="color: #666666">.</span>shape)
|
||||
|
||||
error <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
variance <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
polydegree <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(x, y, test_size<span style="color: #666666">=0.2</span>)
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
scaler<span style="color: #666666">.</span>fit(X_train)
|
||||
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
||||
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> degree <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1</span>,maxdegree):
|
||||
model <span style="color: #666666">=</span> GradientBoostingRegressor(max_depth<span style="color: #666666">=</span>degree, n_estimators<span style="color: #666666">=100</span>, learning_rate<span style="color: #666666">=1.0</span>)
|
||||
model<span style="color: #666666">.</span>fit(X_train_scaled,y_train)
|
||||
y_pred <span style="color: #666666">=</span> model<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
polydegree[degree] <span style="color: #666666">=</span> degree
|
||||
error[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>mean((y_test <span style="color: #666666">-</span> y_pred)<span style="color: #666666">**2</span>) )
|
||||
bias[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( (y_test <span style="color: #666666">-</span> np<span style="color: #666666">.</span>mean(y_pred))<span style="color: #666666">**2</span> )
|
||||
variance[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>var(y_pred) )
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Max depth:'</span>, degree)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Error:'</span>, error[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Bias^2:'</span>, bias[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Var:'</span>, variance[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'</span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> >= </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> + </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> = </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121">'</span><span style="color: #666666">.</span>format(error[degree], bias[degree], variance[degree], bias[degree]<span style="color: #666666">+</span>variance[degree]))
|
||||
|
||||
plt<span style="color: #666666">.</span>xlim(<span style="color: #666666">1</span>,maxdegree<span style="color: #666666">-1</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, error, label<span style="color: #666666">=</span><span style="color: #BA2121">'Error'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, bias, label<span style="color: #666666">=</span><span style="color: #BA2121">'bias'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, variance, label<span style="color: #666666">=</span><span style="color: #BA2121">'Variance'</span>)
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
save_fig(<span style="color: #BA2121">"gdregression"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -251,6 +225,11 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._week45-bs025.html">26</a></li>
|
||||
<li><a href="._week45-bs026.html">27</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs028.html">29</a></li>
|
||||
<li><a href="._week45-bs029.html">30</a></li>
|
||||
<li><a href="._week45-bs030.html">31</a></li>
|
||||
<li><a href="._week45-bs031.html">32</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs024.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,51 +193,38 @@ MathJax.Hub.Config({
|
||||
<a name="part0024"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec23" class="anchor">Gradient Boosting, Classification Example </h2>
|
||||
<h2 id="___sec23" class="anchor">The Squared-Error again! Steepest Descent </h2>
|
||||
|
||||
<p>
|
||||
We start again with our cost function \( {\cal C}(\boldsymbol{y}m\boldsymbol{f})=\sum_{i=0}^{n-1}{\cal L}(y_i, f(x_i)) \) where we want to minimize
|
||||
This means that for every iteration, we need to optimize
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> GradientBoostingClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
|
||||
$$
|
||||
(\hat{\boldsymbol{f}}) = \mathrm{argmin}_{\boldsymbol{f}}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f(x_i))^2.
|
||||
$$
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Load the data</span>
|
||||
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
||||
<p>
|
||||
We define a real function \( h_m(x) \) that defines our final function \( f_M(x) \) as
|
||||
$$
|
||||
f_M(x) = \sum_{m=0}^M h_m(x).
|
||||
$$
|
||||
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
||||
<span style="color: #008000">print</span>(X_train<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #008000">print</span>(X_test<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic">#now scale the data</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
scaler<span style="color: #666666">.</span>fit(X_train)
|
||||
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
||||
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
||||
<p>
|
||||
In the steepest decent approach we approximate \( h_m(x) = -\rho_m g_m(x) \), where \( \rho_m \) is a scalar and \( g_m(x) \) the gradient defined as
|
||||
$$
|
||||
g_m(x_i) = \left[ \frac{\partial {\cal L}(y_i, f(x_i))}{\partial f(x_i)}\right]_{f(x_i)=f_{m-1}(x_i)}.
|
||||
$$
|
||||
|
||||
gd_clf <span style="color: #666666">=</span> GradientBoostingClassifier(max_depth<span style="color: #666666">=3</span>, n_estimators<span style="color: #666666">=100</span>, learning_rate<span style="color: #666666">=1.0</span>)
|
||||
gd_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #408080; font-style: italic">#Cross validation</span>
|
||||
accuracy <span style="color: #666666">=</span> cross_validate(gd_clf,X_test_scaled,y_test,cv<span style="color: #666666">=10</span>)[<span style="color: #BA2121">'test_score'</span>]
|
||||
<span style="color: #008000">print</span>(accuracy)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Random Forests and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(gd_clf<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
<p>
|
||||
With the new gradient we can update \( f_m(x) = f_{m-1}(x) -\rho_m g_m(x) \). Using the above squared-error function we see that
|
||||
the gradient is \( g_m(x_i) = -2(y_i-f(x_i)) \).
|
||||
|
||||
<p>
|
||||
Choosing \( f_0(x)=0 \) we obtain \( g_m(x) = -2y_i \) and inserting this into the minimization problem for the cost function we have
|
||||
$$
|
||||
(\rho_1) = \mathrm{argmin}_{\rho}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i+2\rho y_i)^2.
|
||||
$$
|
||||
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
y_pred <span style="color: #666666">=</span> gd_clf<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
save_fig(<span style="color: #BA2121">"gdclassiffierconfusion"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
y_probas <span style="color: #666666">=</span> gd_clf<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
||||
save_fig(<span style="color: #BA2121">"gdclassiffierroc"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
||||
save_fig(<span style="color: #BA2121">"gdclassiffiercgain"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -244,6 +244,11 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._week45-bs025.html">26</a></li>
|
||||
<li><a href="._week45-bs026.html">27</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs028.html">29</a></li>
|
||||
<li><a href="._week45-bs029.html">30</a></li>
|
||||
<li><a href="._week45-bs030.html">31</a></li>
|
||||
<li><a href="._week45-bs031.html">32</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs025.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -180,23 +193,20 @@ MathJax.Hub.Config({
|
||||
<a name="part0025"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec24" class="anchor">XGBoost: Extreme Gradient Boosting </h2>
|
||||
<h2 id="___sec24" class="anchor">Steepest Descent Example </h2>
|
||||
|
||||
<p>
|
||||
<a href="https://github.com/dmlc/xgboost" target="_self">XGBoost</a> or Extreme Gradient
|
||||
Boosting, is an optimized distributed gradient boosting library
|
||||
designed to be highly efficient, flexible and portable. It implements
|
||||
machine learning algorithms under the Gradient Boosting
|
||||
framework. XGBoost provides a parallel tree boosting that solve many
|
||||
data science problems in a fast and accurate way. See the <a href="https://arxiv.org/abs/1603.02754" target="_self">article by Chen and Guestrin</a>.
|
||||
Optimizing with respect to \( \rho \) we obtain (taking the derivative) that \( \rho_1 = -1/2 \). We have then that
|
||||
$$
|
||||
f_1(x) = f_{0}(x) -\rho_1 g_1(x)=-y_i.
|
||||
$$
|
||||
|
||||
<p>
|
||||
The authors design and build a highly scalable end-to-end tree
|
||||
boosting system. It has a theoretically justified weighted quantile
|
||||
sketch for efficient proposal calculation. It introduces a novel sparsity-aware algorithm for parallel tree learning and an effective cache-aware block structure for out-of-core tree learning.
|
||||
We can then proceed and compute
|
||||
$$
|
||||
g_2(x_i) = \left[ \frac{\partial {\cal L}(y_i, f(x_i))}{\partial f(x_i)}\right]_{f(x_i)=f_{1}(x_i)=y_i}=-4y_i,
|
||||
$$
|
||||
|
||||
<p>
|
||||
It is now the algorithm which wins essentially all ML competitions!!!
|
||||
and find a new value for \( \rho_2=-1/2 \) and continue till we have reached \( m=M \). We can modify the steepest descent method, or steepest boosting, by introducing what is called <b>gradient boosting</b>.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -216,6 +226,11 @@ It is now the algorithm which wins essentially all ML competitions!!!
|
||||
<li class="active"><a href="._week45-bs025.html">26</a></li>
|
||||
<li><a href="._week45-bs026.html">27</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs028.html">29</a></li>
|
||||
<li><a href="._week45-bs029.html">30</a></li>
|
||||
<li><a href="._week45-bs030.html">31</a></li>
|
||||
<li><a href="._week45-bs031.html">32</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs026.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -43,64 +43,72 @@ Automatically generated HTML file from DocOnce source
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -140,31 +148,36 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#___sec0" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#___sec2" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#___sec3" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#___sec4" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#___sec5" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#___sec6" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#___sec7" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#___sec8" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#___sec9" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#___sec10" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#___sec11" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#___sec12" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#___sec13" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#___sec14" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#___sec15" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#___sec16" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#___sec17" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#___sec18" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#___sec19" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#___sec20" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#___sec21" style="font-size: 80%;">AdaBoost for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#___sec22" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#___sec23" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#___sec24" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#___sec25" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#___sec26" style="font-size: 80%;">Gradient Boosting Example, Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#___sec27" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#___sec28" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#___sec29" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#___sec30" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#___sec31" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -199,7 +212,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Oct 31, 2020</h4></center> <!-- date -->
|
||||
<center><h4>Nov 2, 2020</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
|
||||
@@ -223,7 +236,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._week45-bs008.html">9</a></li>
|
||||
<li><a href="._week45-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs032.html">33</a></li>
|
||||
<li><a href="._week45-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -148,7 +148,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p> <br>
|
||||
<center><h4>Oct 31, 2020</h4></center> <!-- date -->
|
||||
<center><h4>Nov 2, 2020</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
|
||||
@@ -162,12 +162,12 @@ MathJax.Hub.Config({
|
||||
<h2 id="___sec0">Overview of week 45 </h2>
|
||||
|
||||
<ul>
|
||||
<p><li> "Thursday: Wrapping up from last week. Bagging and Random forests.</li>
|
||||
<p><li> "Thursday: Wrapping up from last week. Bagging and Random forests. Boosting methods.</li>
|
||||
<p><li> "Friday: Boosting and gradient boosting</li>
|
||||
</ul>
|
||||
<p>
|
||||
|
||||
Geron's chapter 7. See also lecture from <a href="https://www.uio.no/studier/emner/matnat/math/STK-IN4300/h20/slides/lecture_7.pdf" target="_blank">STK-IN4300, lecture 7</a>. Chapter 9.2 of Hastie et al contains also a good discussion.
|
||||
Geron's chapter 7. See also lecture from <a href="https://www.uio.no/studier/emner/matnat/math/STK-IN4300/h20/slides/lecture_9.pdf" target="_blank">STK-IN4300, lecture 9</a>. Chapter 10 (sections 10.1-10.10 are the most relevant ones) of Hastie et al contains also a good discussion.
|
||||
</section>
|
||||
|
||||
|
||||
@@ -176,11 +176,193 @@ Geron's chapter 7. See also lecture from <a href="https://www.uio.no/studier/emn
|
||||
|
||||
<p>
|
||||
Bagging, voting and random forests.
|
||||
The material on bagging and voting is a repeat from last week and can be found in the slides from week 44.
|
||||
We repeat here the voting approach since this will serve as a motivation for boosting methods later.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec2">Random forests </h2>
|
||||
<h2 id="___sec2">Why Voting? </h2>
|
||||
|
||||
<p>
|
||||
The idea behind boosting, and voting as well can be phrased as follows:
|
||||
<b>Can a group of people somehow arrive at highly
|
||||
reasoned decisions, despite the weak judgement of the individual
|
||||
members?</b>
|
||||
|
||||
<p>
|
||||
The aim is to create a good classifier by combining several weak classifiers.
|
||||
<b>A weak classifier is a classifier which is able to produce results that are only slightly better than guessing at random.</b>
|
||||
|
||||
<p>
|
||||
The basic approach is to apply repeatedly (in boosting this is done in an iterative way) a weak classifier to modifications of the data.
|
||||
In voting we simply apply the law of large numbers while in boosting we give more weight to misclassified data in
|
||||
each iteration.
|
||||
|
||||
<p>
|
||||
Decision trees play an important role as our weak classifier. They serve as the basic method.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec3">Tossing coins </h2>
|
||||
The simplest case is a so-called voting ensemble. To illustrate this, Think of you tossing coins with a biased outcome of 51 per cent for heads and 49% for tails.
|
||||
With only few tosses, you may not clearly see this distribution. However, after some thousands of tosses (sounds like you may have some spare time problems), there will be a clear majority of heads.
|
||||
With 2000 tosses you should see approximately 1020 heads and 980 tails.
|
||||
|
||||
<p>
|
||||
We can then state that the outcome is a clear majority of heads. If you do this ten thousand times, it is easy to see that there is a 97% likelihood of a majority of heads.
|
||||
|
||||
<p>
|
||||
Another example would be to collect all polls before an
|
||||
election. Different polls may show different likelihoods for a
|
||||
candidate winning with say a majority of the popular vote. The majority vote
|
||||
would then consist in many polls indicating that this candidate will
|
||||
actually win.
|
||||
|
||||
<p>
|
||||
The example here shows how we can implement the coin tossing case, clealry demostrating that after some tosses we see the law of large numbers kicking in.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec4">Simple Voting Example, head or tail </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span>heads_proba = <span style="color: #B452CD">0.51</span>
|
||||
coin_tosses = (np.random.rand(<span style="color: #B452CD">10000</span>, <span style="color: #B452CD">10</span>) < heads_proba).astype(np.int32)
|
||||
cumulative_heads_ratio = np.cumsum(coin_tosses, axis=<span style="color: #B452CD">0</span>) / np.arange(<span style="color: #B452CD">1</span>, <span style="color: #B452CD">10001</span>).reshape(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>)
|
||||
plt.figure(figsize=(<span style="color: #B452CD">8</span>,<span style="color: #B452CD">3.5</span>))
|
||||
plt.plot(cumulative_heads_ratio)
|
||||
plt.plot([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">10000</span>], [<span style="color: #B452CD">0.51</span>, <span style="color: #B452CD">0.51</span>], <span style="color: #CD5555">"k--"</span>, linewidth=<span style="color: #B452CD">2</span>, label=<span style="color: #CD5555">"51%"</span>)
|
||||
plt.plot([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">10000</span>], [<span style="color: #B452CD">0.5</span>, <span style="color: #B452CD">0.5</span>], <span style="color: #CD5555">"k-"</span>, label=<span style="color: #CD5555">"50%"</span>)
|
||||
plt.xlabel(<span style="color: #CD5555">"Number of coin tosses"</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">"Heads ratio"</span>)
|
||||
plt.legend(loc=<span style="color: #CD5555">"lower right"</span>)
|
||||
plt.axis([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">10000</span>, <span style="color: #B452CD">0.42</span>, <span style="color: #B452CD">0.58</span>])
|
||||
save_fig(<span style="color: #CD5555">"votingsimple"</span>)
|
||||
plt.show()
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec5">Using the Voting Classifier </h2>
|
||||
|
||||
<p>
|
||||
We can use the voting classifier on other data sets, here the excting binary case of two distinct objects using the make moons functionality of -Scikit-Learn-.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_moons
|
||||
|
||||
X, y = make_moons(n_samples=<span style="color: #B452CD">500</span>, noise=<span style="color: #B452CD">0.30</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> VotingClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||||
|
||||
log_clf = LogisticRegression(solver=<span style="color: #CD5555">"liblinear"</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf = RandomForestClassifier(n_estimators=<span style="color: #B452CD">10</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
svm_clf = SVC(gamma=<span style="color: #CD5555">"auto"</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[(<span style="color: #CD5555">'lr'</span>, log_clf), (<span style="color: #CD5555">'rf'</span>, rnd_clf), (<span style="color: #CD5555">'svc'</span>, svm_clf)],
|
||||
voting=<span style="color: #CD5555">'hard'</span>)
|
||||
|
||||
voting_clf.fit(X_train, y_train)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> clf <span style="color: #8B008B">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
<span style="color: #658b00">print</span>(clf.<span style="color: #00688B">__class__</span>.<span style="color: #00688B">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
|
||||
log_clf = LogisticRegression(solver=<span style="color: #CD5555">"liblinear"</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf = RandomForestClassifier(n_estimators=<span style="color: #B452CD">10</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
svm_clf = SVC(gamma=<span style="color: #CD5555">"auto"</span>, probability=<span style="color: #8B008B; font-weight: bold">True</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[(<span style="color: #CD5555">'lr'</span>, log_clf), (<span style="color: #CD5555">'rf'</span>, rnd_clf), (<span style="color: #CD5555">'svc'</span>, svm_clf)],
|
||||
voting=<span style="color: #CD5555">'soft'</span>)
|
||||
voting_clf.fit(X_train, y_train)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> clf <span style="color: #8B008B">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
<span style="color: #658b00">print</span>(clf.<span style="color: #00688B">__class__</span>.<span style="color: #00688B">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec6">Please, not the moons again! Voting and Bagging </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_moons
|
||||
|
||||
X, y = make_moons(n_samples=<span style="color: #B452CD">500</span>, noise=<span style="color: #B452CD">0.30</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=<span style="color: #B452CD">42</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> VotingClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||||
|
||||
log_clf = LogisticRegression(random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf = RandomForestClassifier(random_state=<span style="color: #B452CD">42</span>)
|
||||
svm_clf = SVC(random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[(<span style="color: #CD5555">'lr'</span>, log_clf), (<span style="color: #CD5555">'rf'</span>, rnd_clf), (<span style="color: #CD5555">'svc'</span>, svm_clf)],
|
||||
voting=<span style="color: #CD5555">'hard'</span>)
|
||||
voting_clf.fit(X_train, y_train)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> clf <span style="color: #8B008B">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
<span style="color: #658b00">print</span>(clf.<span style="color: #00688B">__class__</span>.<span style="color: #00688B">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span>log_clf = LogisticRegression(random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf = RandomForestClassifier(random_state=<span style="color: #B452CD">42</span>)
|
||||
svm_clf = SVC(probability=<span style="color: #8B008B; font-weight: bold">True</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[(<span style="color: #CD5555">'lr'</span>, log_clf), (<span style="color: #CD5555">'rf'</span>, rnd_clf), (<span style="color: #CD5555">'svc'</span>, svm_clf)],
|
||||
voting=<span style="color: #CD5555">'soft'</span>)
|
||||
voting_clf.fit(X_train, y_train)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> clf <span style="color: #8B008B">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
<span style="color: #658b00">print</span>(clf.<span style="color: #00688B">__class__</span>.<span style="color: #00688B">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec7">Random forests </h2>
|
||||
|
||||
<p>
|
||||
Random forests provide an improvement over bagged trees by way of a
|
||||
@@ -226,7 +408,7 @@ this setting.
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec3">Random Forest Algorithm </h2>
|
||||
<h2 id="___sec8">Random Forest Algorithm </h2>
|
||||
The algorithm described here can be applied to both classification and regression problems.
|
||||
|
||||
<p>
|
||||
@@ -257,7 +439,7 @@ We will grow of forest of say \( B \) trees.
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec4">Random Forests Compared with other Methods on the Cancer Data </h2>
|
||||
<h2 id="___sec9">Random Forests Compared with other Methods on the Cancer Data </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -331,7 +513,7 @@ plt.show()
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec5">Compare Bagging on Trees with Random Forests </h2>
|
||||
<h2 id="___sec10">Compare Bagging on Trees with Random Forests </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -354,7 +536,7 @@ np.sum(y_pred == y_pred_rf) / <span style="color: #658b00">len</span>(y_pred)
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec6">Boosting, a Bird's Eye View </h2>
|
||||
<h2 id="___sec11">Boosting, a Bird's Eye View </h2>
|
||||
|
||||
<p>
|
||||
The basic idea is to combine weak classifiers in order to create a good
|
||||
@@ -371,7 +553,7 @@ them with a factor.
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec7">What is boosting? Additive Modelling/Iterative Fitting </h2>
|
||||
<h2 id="___sec12">What is boosting? Additive Modelling/Iterative Fitting </h2>
|
||||
|
||||
<p>
|
||||
Boosting is a way of fitting an additive expansion in a set of
|
||||
@@ -431,7 +613,7 @@ In iterative fitting or additive modeling, we minimize the cost function with re
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec8">Iterative Fitting, Regression and Squared-error Cost Function </h2>
|
||||
<h2 id="___sec13">Iterative Fitting, Regression and Squared-error Cost Function </h2>
|
||||
|
||||
<p>
|
||||
The way we proceed is as follows (here we specialize to the squared-error cost function)
|
||||
@@ -457,7 +639,7 @@ at the internal nodes, and the predictions at the terminal nodes.
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec9">Squared-Error Example and Iterative Fitting </h2>
|
||||
<h2 id="___sec14">Squared-Error Example and Iterative Fitting </h2>
|
||||
|
||||
<p>
|
||||
To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function.
|
||||
@@ -515,7 +697,7 @@ The solution to these two equations gives us in turn \( \beta_1 \) and \( \gamma
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec10">Iterative Fitting, Classification and AdaBoost </h2>
|
||||
<h2 id="___sec15">Iterative Fitting, Classification and AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
Let us consider a binary classification problem with two outcomes \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
|
||||
@@ -556,7 +738,7 @@ $$
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec11">Adaptive Boosting, AdaBoost </h2>
|
||||
<h2 id="___sec16">Adaptive Boosting, AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
In our iterative procedure we define thus
|
||||
@@ -590,7 +772,7 @@ where we have defined \( w_i^m= \exp{(-y_if_{m-1}(x_i))} \).
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec12">Building up AdaBoost </h2>
|
||||
<h2 id="___sec17">Building up AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
First, for any \( \beta > 0 \), we optimize \( G \) by setting
|
||||
@@ -648,7 +830,7 @@ $$
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec13">Adaptive boosting: AdaBoost, Basic Algorithm </h2>
|
||||
<h2 id="___sec18">Adaptive boosting: AdaBoost, Basic Algorithm </h2>
|
||||
|
||||
<p>
|
||||
The algorithm here is rather straightforward. Assume that our weak
|
||||
@@ -672,7 +854,7 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec14">Basic Steps of AdaBoost </h2>
|
||||
<h2 id="___sec19">Basic Steps of AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
With the above definitions we are now ready to set up the algorithm for AdaBoost.
|
||||
@@ -713,7 +895,7 @@ observations that are missed in the previous iterations.
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec15">AdaBoost Examples </h2>
|
||||
<h2 id="___sec20">AdaBoost Examples </h2>
|
||||
|
||||
<p>
|
||||
Using <b>Scikit-Learn</b> it is easy to apply the adaptive boosting algorithm, as done here.
|
||||
@@ -747,7 +929,7 @@ plt.show()
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec16">AdaBoost for Regression </h2>
|
||||
<h2 id="___sec21">AdaBoost for Regression </h2>
|
||||
|
||||
<p>
|
||||
Here we present <a href="https://pdfs.semanticscholar.org/8d49/e2dedb817f2c3330e74b63c5fc86d2399ce3.pdf" target="_blank">Drucker's AdaBoost</a> tailored for regression.
|
||||
@@ -776,7 +958,7 @@ Start by selecting a set of training data \( n \) and assign to each entry a wei
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec17">Gradient boosting: Basics with Steepest Descent </h2>
|
||||
<h2 id="___sec22">Gradient boosting: Basics with Steepest Descent </h2>
|
||||
|
||||
<p>
|
||||
Gradient boosting is again a similar technique to Adaptive boosting,
|
||||
@@ -791,7 +973,7 @@ function was the least squares function.
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec18">The Squared-Error again! Steepest Descent </h2>
|
||||
<h2 id="___sec23">The Squared-Error again! Steepest Descent </h2>
|
||||
|
||||
<p>
|
||||
We start again with our cost function \( {\cal C}(\boldsymbol{y}m\boldsymbol{f})=\sum_{i=0}^{n-1}{\cal L}(y_i, f(x_i)) \) where we want to minimize
|
||||
@@ -834,7 +1016,7 @@ $$
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec19">Steepest Descent Example </h2>
|
||||
<h2 id="___sec24">Steepest Descent Example </h2>
|
||||
|
||||
<p>
|
||||
Optimizing with respect to \( \rho \) we obtain (taking the derivative) that \( \rho_1 = -1/2 \). We have then that
|
||||
@@ -856,7 +1038,7 @@ and find a new value for \( \rho_2=-1/2 \) and continue till we have reached \(
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec20">Gradient Boosting, algorithm </h2>
|
||||
<h2 id="___sec25">Gradient Boosting, algorithm </h2>
|
||||
|
||||
<p>
|
||||
Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard squared-error function
|
||||
@@ -884,7 +1066,7 @@ The way we proceed in an iterative fashion is to
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec21">Gradient Boosting Example, Regression </h2>
|
||||
<h2 id="___sec26">Gradient Boosting Example, Regression </h2>
|
||||
|
||||
<p>
|
||||
We discuss here the difference between the steepest descent approach and gradient boosting by repeating our simple regression example above.
|
||||
@@ -892,7 +1074,7 @@ We discuss here the difference between the steepest descent approach and gradien
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec22">Gradient Boosting, Examples of Regression </h2>
|
||||
<h2 id="___sec27">Gradient Boosting, Examples of Regression </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -947,7 +1129,7 @@ plt.show()
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec23">Gradient Boosting, Classification Example </h2>
|
||||
<h2 id="___sec28">Gradient Boosting, Classification Example </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -996,7 +1178,7 @@ plt.show()
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec24">XGBoost: Extreme Gradient Boosting </h2>
|
||||
<h2 id="___sec29">XGBoost: Extreme Gradient Boosting </h2>
|
||||
|
||||
<p>
|
||||
<a href="https://github.com/dmlc/xgboost" target="_blank">XGBoost</a> or Extreme Gradient
|
||||
@@ -1017,7 +1199,7 @@ It is now the algorithm which wins essentially all ML competitions!!!
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec25">Regression Case </h2>
|
||||
<h2 id="___sec30">Regression Case </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -1073,7 +1255,7 @@ plt.show()
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec26">Xgboost on the Cancer Data </h2>
|
||||
<h2 id="___sec31">Xgboost on the Cancer Data </h2>
|
||||
|
||||
<p>
|
||||
As you will see from the confusion matrix below, XGBoots does an excellent job on the Wisconsin cancer data and outperforms essentially all agorithms we have discussed till now.
|
||||
|
||||
@@ -37,64 +37,72 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -136,7 +144,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Oct 31, 2020</h4></center> <!-- date -->
|
||||
<center><h4>Nov 2, 2020</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
@@ -144,11 +152,11 @@ MathJax.Hub.Config({
|
||||
<h2 id="___sec0">Overview of week 45 </h2>
|
||||
|
||||
<ul>
|
||||
<li> "Thursday: Wrapping up from last week. Bagging and Random forests.</li>
|
||||
<li> "Thursday: Wrapping up from last week. Bagging and Random forests. Boosting methods.</li>
|
||||
<li> "Friday: Boosting and gradient boosting</li>
|
||||
</ul>
|
||||
|
||||
Geron's chapter 7. See also lecture from <a href="https://www.uio.no/studier/emner/matnat/math/STK-IN4300/h20/slides/lecture_7.pdf" target="_blank">STK-IN4300, lecture 7</a>. Chapter 9.2 of Hastie et al contains also a good discussion.
|
||||
Geron's chapter 7. See also lecture from <a href="https://www.uio.no/studier/emner/matnat/math/STK-IN4300/h20/slides/lecture_9.pdf" target="_blank">STK-IN4300, lecture 9</a>. Chapter 10 (sections 10.1-10.10 are the most relevant ones) of Hastie et al contains also a good discussion.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
@@ -157,11 +165,190 @@ Geron's chapter 7. See also lecture from <a href="https://www.uio.no/studier/emn
|
||||
|
||||
<p>
|
||||
Bagging, voting and random forests.
|
||||
The material on bagging and voting is a repeat from last week and can be found in the slides from week 44.
|
||||
We repeat here the voting approach since this will serve as a motivation for boosting methods later.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec2">Random forests </h2>
|
||||
<h2 id="___sec2">Why Voting? </h2>
|
||||
|
||||
<p>
|
||||
The idea behind boosting, and voting as well can be phrased as follows:
|
||||
<b>Can a group of people somehow arrive at highly
|
||||
reasoned decisions, despite the weak judgement of the individual
|
||||
members?</b>
|
||||
|
||||
<p>
|
||||
The aim is to create a good classifier by combining several weak classifiers.
|
||||
<b>A weak classifier is a classifier which is able to produce results that are only slightly better than guessing at random.</b>
|
||||
|
||||
<p>
|
||||
The basic approach is to apply repeatedly (in boosting this is done in an iterative way) a weak classifier to modifications of the data.
|
||||
In voting we simply apply the law of large numbers while in boosting we give more weight to misclassified data in
|
||||
each iteration.
|
||||
|
||||
<p>
|
||||
Decision trees play an important role as our weak classifier. They serve as the basic method.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec3">Tossing coins </h2>
|
||||
The simplest case is a so-called voting ensemble. To illustrate this, Think of you tossing coins with a biased outcome of 51 per cent for heads and 49% for tails.
|
||||
With only few tosses, you may not clearly see this distribution. However, after some thousands of tosses (sounds like you may have some spare time problems), there will be a clear majority of heads.
|
||||
With 2000 tosses you should see approximately 1020 heads and 980 tails.
|
||||
|
||||
<p>
|
||||
We can then state that the outcome is a clear majority of heads. If you do this ten thousand times, it is easy to see that there is a 97% likelihood of a majority of heads.
|
||||
|
||||
<p>
|
||||
Another example would be to collect all polls before an
|
||||
election. Different polls may show different likelihoods for a
|
||||
candidate winning with say a majority of the popular vote. The majority vote
|
||||
would then consist in many polls indicating that this candidate will
|
||||
actually win.
|
||||
|
||||
<p>
|
||||
The example here shows how we can implement the coin tossing case, clealry demostrating that after some tosses we see the law of large numbers kicking in.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec4">Simple Voting Example, head or tail </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span>heads_proba = <span style="color: #B452CD">0.51</span>
|
||||
coin_tosses = (np.random.rand(<span style="color: #B452CD">10000</span>, <span style="color: #B452CD">10</span>) < heads_proba).astype(np.int32)
|
||||
cumulative_heads_ratio = np.cumsum(coin_tosses, axis=<span style="color: #B452CD">0</span>) / np.arange(<span style="color: #B452CD">1</span>, <span style="color: #B452CD">10001</span>).reshape(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>)
|
||||
plt.figure(figsize=(<span style="color: #B452CD">8</span>,<span style="color: #B452CD">3.5</span>))
|
||||
plt.plot(cumulative_heads_ratio)
|
||||
plt.plot([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">10000</span>], [<span style="color: #B452CD">0.51</span>, <span style="color: #B452CD">0.51</span>], <span style="color: #CD5555">"k--"</span>, linewidth=<span style="color: #B452CD">2</span>, label=<span style="color: #CD5555">"51%"</span>)
|
||||
plt.plot([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">10000</span>], [<span style="color: #B452CD">0.5</span>, <span style="color: #B452CD">0.5</span>], <span style="color: #CD5555">"k-"</span>, label=<span style="color: #CD5555">"50%"</span>)
|
||||
plt.xlabel(<span style="color: #CD5555">"Number of coin tosses"</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">"Heads ratio"</span>)
|
||||
plt.legend(loc=<span style="color: #CD5555">"lower right"</span>)
|
||||
plt.axis([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">10000</span>, <span style="color: #B452CD">0.42</span>, <span style="color: #B452CD">0.58</span>])
|
||||
save_fig(<span style="color: #CD5555">"votingsimple"</span>)
|
||||
plt.show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec5">Using the Voting Classifier </h2>
|
||||
|
||||
<p>
|
||||
We can use the voting classifier on other data sets, here the excting binary case of two distinct objects using the make moons functionality of -Scikit-Learn-.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_moons
|
||||
|
||||
X, y = make_moons(n_samples=<span style="color: #B452CD">500</span>, noise=<span style="color: #B452CD">0.30</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> VotingClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||||
|
||||
log_clf = LogisticRegression(solver=<span style="color: #CD5555">"liblinear"</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf = RandomForestClassifier(n_estimators=<span style="color: #B452CD">10</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
svm_clf = SVC(gamma=<span style="color: #CD5555">"auto"</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[(<span style="color: #CD5555">'lr'</span>, log_clf), (<span style="color: #CD5555">'rf'</span>, rnd_clf), (<span style="color: #CD5555">'svc'</span>, svm_clf)],
|
||||
voting=<span style="color: #CD5555">'hard'</span>)
|
||||
|
||||
voting_clf.fit(X_train, y_train)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> clf <span style="color: #8B008B">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
<span style="color: #658b00">print</span>(clf.<span style="color: #00688B">__class__</span>.<span style="color: #00688B">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
|
||||
log_clf = LogisticRegression(solver=<span style="color: #CD5555">"liblinear"</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf = RandomForestClassifier(n_estimators=<span style="color: #B452CD">10</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
svm_clf = SVC(gamma=<span style="color: #CD5555">"auto"</span>, probability=<span style="color: #8B008B; font-weight: bold">True</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[(<span style="color: #CD5555">'lr'</span>, log_clf), (<span style="color: #CD5555">'rf'</span>, rnd_clf), (<span style="color: #CD5555">'svc'</span>, svm_clf)],
|
||||
voting=<span style="color: #CD5555">'soft'</span>)
|
||||
voting_clf.fit(X_train, y_train)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> clf <span style="color: #8B008B">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
<span style="color: #658b00">print</span>(clf.<span style="color: #00688B">__class__</span>.<span style="color: #00688B">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec6">Please, not the moons again! Voting and Bagging </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_moons
|
||||
|
||||
X, y = make_moons(n_samples=<span style="color: #B452CD">500</span>, noise=<span style="color: #B452CD">0.30</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=<span style="color: #B452CD">42</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> VotingClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||||
|
||||
log_clf = LogisticRegression(random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf = RandomForestClassifier(random_state=<span style="color: #B452CD">42</span>)
|
||||
svm_clf = SVC(random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[(<span style="color: #CD5555">'lr'</span>, log_clf), (<span style="color: #CD5555">'rf'</span>, rnd_clf), (<span style="color: #CD5555">'svc'</span>, svm_clf)],
|
||||
voting=<span style="color: #CD5555">'hard'</span>)
|
||||
voting_clf.fit(X_train, y_train)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> clf <span style="color: #8B008B">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
<span style="color: #658b00">print</span>(clf.<span style="color: #00688B">__class__</span>.<span style="color: #00688B">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span>log_clf = LogisticRegression(random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf = RandomForestClassifier(random_state=<span style="color: #B452CD">42</span>)
|
||||
svm_clf = SVC(probability=<span style="color: #8B008B; font-weight: bold">True</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[(<span style="color: #CD5555">'lr'</span>, log_clf), (<span style="color: #CD5555">'rf'</span>, rnd_clf), (<span style="color: #CD5555">'svc'</span>, svm_clf)],
|
||||
voting=<span style="color: #CD5555">'soft'</span>)
|
||||
voting_clf.fit(X_train, y_train)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> clf <span style="color: #8B008B">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
<span style="color: #658b00">print</span>(clf.<span style="color: #00688B">__class__</span>.<span style="color: #00688B">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec7">Random forests </h2>
|
||||
|
||||
<p>
|
||||
Random forests provide an improvement over bagged trees by way of a
|
||||
@@ -205,7 +392,7 @@ this setting.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec3">Random Forest Algorithm </h2>
|
||||
<h2 id="___sec8">Random Forest Algorithm </h2>
|
||||
The algorithm described here can be applied to both classification and regression problems.
|
||||
|
||||
<p>
|
||||
@@ -231,7 +418,7 @@ We will grow of forest of say \( B \) trees.
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec4">Random Forests Compared with other Methods on the Cancer Data </h2>
|
||||
<h2 id="___sec9">Random Forests Compared with other Methods on the Cancer Data </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -304,7 +491,7 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec5">Compare Bagging on Trees with Random Forests </h2>
|
||||
<h2 id="___sec10">Compare Bagging on Trees with Random Forests </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -326,7 +513,7 @@ np.sum(y_pred == y_pred_rf) / <span style="color: #658b00">len</span>(y_pred)
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec6">Boosting, a Bird's Eye View </h2>
|
||||
<h2 id="___sec11">Boosting, a Bird's Eye View </h2>
|
||||
|
||||
<p>
|
||||
The basic idea is to combine weak classifiers in order to create a good
|
||||
@@ -343,7 +530,7 @@ them with a factor.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec7">What is boosting? Additive Modelling/Iterative Fitting </h2>
|
||||
<h2 id="___sec12">What is boosting? Additive Modelling/Iterative Fitting </h2>
|
||||
|
||||
<p>
|
||||
Boosting is a way of fitting an additive expansion in a set of
|
||||
@@ -395,7 +582,7 @@ In iterative fitting or additive modeling, we minimize the cost function with re
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec8">Iterative Fitting, Regression and Squared-error Cost Function </h2>
|
||||
<h2 id="___sec13">Iterative Fitting, Regression and Squared-error Cost Function </h2>
|
||||
|
||||
<p>
|
||||
The way we proceed is as follows (here we specialize to the squared-error cost function)
|
||||
@@ -420,7 +607,7 @@ at the internal nodes, and the predictions at the terminal nodes.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec9">Squared-Error Example and Iterative Fitting </h2>
|
||||
<h2 id="___sec14">Squared-Error Example and Iterative Fitting </h2>
|
||||
|
||||
<p>
|
||||
To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function.
|
||||
@@ -468,7 +655,7 @@ The solution to these two equations gives us in turn \( \beta_1 \) and \( \gamma
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec10">Iterative Fitting, Classification and AdaBoost </h2>
|
||||
<h2 id="___sec15">Iterative Fitting, Classification and AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
Let us consider a binary classification problem with two outcomes \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
|
||||
@@ -503,7 +690,7 @@ $$
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec11">Adaptive Boosting, AdaBoost </h2>
|
||||
<h2 id="___sec16">Adaptive Boosting, AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
In our iterative procedure we define thus
|
||||
@@ -531,7 +718,7 @@ where we have defined \( w_i^m= \exp{(-y_if_{m-1}(x_i))} \).
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec12">Building up AdaBoost </h2>
|
||||
<h2 id="___sec17">Building up AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
First, for any \( \beta > 0 \), we optimize \( G \) by setting
|
||||
@@ -575,7 +762,7 @@ $$
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec13">Adaptive boosting: AdaBoost, Basic Algorithm </h2>
|
||||
<h2 id="___sec18">Adaptive boosting: AdaBoost, Basic Algorithm </h2>
|
||||
|
||||
<p>
|
||||
The algorithm here is rather straightforward. Assume that our weak
|
||||
@@ -597,7 +784,7 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec14">Basic Steps of AdaBoost </h2>
|
||||
<h2 id="___sec19">Basic Steps of AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
With the above definitions we are now ready to set up the algorithm for AdaBoost.
|
||||
@@ -637,7 +824,7 @@ observations that are missed in the previous iterations.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec15">AdaBoost Examples </h2>
|
||||
<h2 id="___sec20">AdaBoost Examples </h2>
|
||||
|
||||
<p>
|
||||
Using <b>Scikit-Learn</b> it is easy to apply the adaptive boosting algorithm, as done here.
|
||||
@@ -670,7 +857,7 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec16">AdaBoost for Regression </h2>
|
||||
<h2 id="___sec21">AdaBoost for Regression </h2>
|
||||
|
||||
<p>
|
||||
Here we present <a href="https://pdfs.semanticscholar.org/8d49/e2dedb817f2c3330e74b63c5fc86d2399ce3.pdf" target="_blank">Drucker's AdaBoost</a> tailored for regression.
|
||||
@@ -698,7 +885,7 @@ Start by selecting a set of training data \( n \) and assign to each entry a wei
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec17">Gradient boosting: Basics with Steepest Descent </h2>
|
||||
<h2 id="___sec22">Gradient boosting: Basics with Steepest Descent </h2>
|
||||
|
||||
<p>
|
||||
Gradient boosting is again a similar technique to Adaptive boosting,
|
||||
@@ -713,7 +900,7 @@ function was the least squares function.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec18">The Squared-Error again! Steepest Descent </h2>
|
||||
<h2 id="___sec23">The Squared-Error again! Steepest Descent </h2>
|
||||
|
||||
<p>
|
||||
We start again with our cost function \( {\cal C}(\boldsymbol{y}m\boldsymbol{f})=\sum_{i=0}^{n-1}{\cal L}(y_i, f(x_i)) \) where we want to minimize
|
||||
@@ -748,7 +935,7 @@ $$
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec19">Steepest Descent Example </h2>
|
||||
<h2 id="___sec24">Steepest Descent Example </h2>
|
||||
|
||||
<p>
|
||||
Optimizing with respect to \( \rho \) we obtain (taking the derivative) that \( \rho_1 = -1/2 \). We have then that
|
||||
@@ -766,7 +953,7 @@ and find a new value for \( \rho_2=-1/2 \) and continue till we have reached \(
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec20">Gradient Boosting, algorithm </h2>
|
||||
<h2 id="___sec25">Gradient Boosting, algorithm </h2>
|
||||
|
||||
<p>
|
||||
Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard squared-error function
|
||||
@@ -792,7 +979,7 @@ The way we proceed in an iterative fashion is to
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec21">Gradient Boosting Example, Regression </h2>
|
||||
<h2 id="___sec26">Gradient Boosting Example, Regression </h2>
|
||||
|
||||
<p>
|
||||
We discuss here the difference between the steepest descent approach and gradient boosting by repeating our simple regression example above.
|
||||
@@ -800,7 +987,7 @@ We discuss here the difference between the steepest descent approach and gradien
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec22">Gradient Boosting, Examples of Regression </h2>
|
||||
<h2 id="___sec27">Gradient Boosting, Examples of Regression </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -854,7 +1041,7 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec23">Gradient Boosting, Classification Example </h2>
|
||||
<h2 id="___sec28">Gradient Boosting, Classification Example </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -902,7 +1089,7 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec24">XGBoost: Extreme Gradient Boosting </h2>
|
||||
<h2 id="___sec29">XGBoost: Extreme Gradient Boosting </h2>
|
||||
|
||||
<p>
|
||||
<a href="https://github.com/dmlc/xgboost" target="_blank">XGBoost</a> or Extreme Gradient
|
||||
@@ -923,7 +1110,7 @@ It is now the algorithm which wins essentially all ML competitions!!!
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec25">Regression Case </h2>
|
||||
<h2 id="___sec30">Regression Case </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -978,7 +1165,7 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec26">Xgboost on the Cancer Data </h2>
|
||||
<h2 id="___sec31">Xgboost on the Cancer Data </h2>
|
||||
|
||||
<p>
|
||||
As you will see from the confusion matrix below, XGBoots does an excellent job on the Wisconsin cancer data and outperforms essentially all agorithms we have discussed till now.
|
||||
|
||||
+240
-53
@@ -42,64 +42,72 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, '___sec0'),
|
||||
('Thursday', 2, None, '___sec1'),
|
||||
('Random forests', 2, None, '___sec2'),
|
||||
('Random Forest Algorithm', 2, None, '___sec3'),
|
||||
('Why Voting?', 2, None, '___sec2'),
|
||||
('Tossing coins', 2, None, '___sec3'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec4'),
|
||||
('Using the Voting Classifier', 2, None, '___sec5'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Random forests', 2, None, '___sec7'),
|
||||
('Random Forest Algorithm', 2, None, '___sec8'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
'___sec9'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec6'),
|
||||
'___sec10'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec11'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec7'),
|
||||
'___sec12'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
'___sec13'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
'___sec14'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec11'),
|
||||
('Building up AdaBoost', 2, None, '___sec12'),
|
||||
'___sec15'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec16'),
|
||||
('Building up AdaBoost', 2, None, '___sec17'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec13'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec14'),
|
||||
('AdaBoost Examples', 2, None, '___sec15'),
|
||||
('AdaBoost for Regression', 2, None, '___sec16'),
|
||||
'___sec18'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec19'),
|
||||
('AdaBoost Examples', 2, None, '___sec20'),
|
||||
('AdaBoost for Regression', 2, None, '___sec21'),
|
||||
('Gradient boosting: Basics with Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
'___sec22'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Steepest Descent Example', 2, None, '___sec19'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec20'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec21'),
|
||||
'___sec23'),
|
||||
('Steepest Descent Example', 2, None, '___sec24'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec25'),
|
||||
('Gradient Boosting Example, Regression', 2, None, '___sec26'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec22'),
|
||||
'___sec27'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec23'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec24'),
|
||||
('Regression Case', 2, None, '___sec25'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec26')]}
|
||||
'___sec28'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec29'),
|
||||
('Regression Case', 2, None, '___sec30'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec31')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -141,7 +149,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Oct 31, 2020</h4></center> <!-- date -->
|
||||
<center><h4>Nov 2, 2020</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
@@ -149,11 +157,11 @@ MathJax.Hub.Config({
|
||||
<h2 id="___sec0">Overview of week 45 </h2>
|
||||
|
||||
<ul>
|
||||
<li> "Thursday: Wrapping up from last week. Bagging and Random forests.</li>
|
||||
<li> "Thursday: Wrapping up from last week. Bagging and Random forests. Boosting methods.</li>
|
||||
<li> "Friday: Boosting and gradient boosting</li>
|
||||
</ul>
|
||||
|
||||
Geron's chapter 7. See also lecture from <a href="https://www.uio.no/studier/emner/matnat/math/STK-IN4300/h20/slides/lecture_7.pdf" target="_blank">STK-IN4300, lecture 7</a>. Chapter 9.2 of Hastie et al contains also a good discussion.
|
||||
Geron's chapter 7. See also lecture from <a href="https://www.uio.no/studier/emner/matnat/math/STK-IN4300/h20/slides/lecture_9.pdf" target="_blank">STK-IN4300, lecture 9</a>. Chapter 10 (sections 10.1-10.10 are the most relevant ones) of Hastie et al contains also a good discussion.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
@@ -162,11 +170,190 @@ Geron's chapter 7. See also lecture from <a href="https://www.uio.no/studier/emn
|
||||
|
||||
<p>
|
||||
Bagging, voting and random forests.
|
||||
The material on bagging and voting is a repeat from last week and can be found in the slides from week 44.
|
||||
We repeat here the voting approach since this will serve as a motivation for boosting methods later.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec2">Random forests </h2>
|
||||
<h2 id="___sec2">Why Voting? </h2>
|
||||
|
||||
<p>
|
||||
The idea behind boosting, and voting as well can be phrased as follows:
|
||||
<b>Can a group of people somehow arrive at highly
|
||||
reasoned decisions, despite the weak judgement of the individual
|
||||
members?</b>
|
||||
|
||||
<p>
|
||||
The aim is to create a good classifier by combining several weak classifiers.
|
||||
<b>A weak classifier is a classifier which is able to produce results that are only slightly better than guessing at random.</b>
|
||||
|
||||
<p>
|
||||
The basic approach is to apply repeatedly (in boosting this is done in an iterative way) a weak classifier to modifications of the data.
|
||||
In voting we simply apply the law of large numbers while in boosting we give more weight to misclassified data in
|
||||
each iteration.
|
||||
|
||||
<p>
|
||||
Decision trees play an important role as our weak classifier. They serve as the basic method.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec3">Tossing coins </h2>
|
||||
The simplest case is a so-called voting ensemble. To illustrate this, Think of you tossing coins with a biased outcome of 51 per cent for heads and 49% for tails.
|
||||
With only few tosses, you may not clearly see this distribution. However, after some thousands of tosses (sounds like you may have some spare time problems), there will be a clear majority of heads.
|
||||
With 2000 tosses you should see approximately 1020 heads and 980 tails.
|
||||
|
||||
<p>
|
||||
We can then state that the outcome is a clear majority of heads. If you do this ten thousand times, it is easy to see that there is a 97% likelihood of a majority of heads.
|
||||
|
||||
<p>
|
||||
Another example would be to collect all polls before an
|
||||
election. Different polls may show different likelihoods for a
|
||||
candidate winning with say a majority of the popular vote. The majority vote
|
||||
would then consist in many polls indicating that this candidate will
|
||||
actually win.
|
||||
|
||||
<p>
|
||||
The example here shows how we can implement the coin tossing case, clealry demostrating that after some tosses we see the law of large numbers kicking in.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec4">Simple Voting Example, head or tail </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>heads_proba <span style="color: #666666">=</span> <span style="color: #666666">0.51</span>
|
||||
coin_tosses <span style="color: #666666">=</span> (np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">10000</span>, <span style="color: #666666">10</span>) <span style="color: #666666"><</span> heads_proba)<span style="color: #666666">.</span>astype(np<span style="color: #666666">.</span>int32)
|
||||
cumulative_heads_ratio <span style="color: #666666">=</span> np<span style="color: #666666">.</span>cumsum(coin_tosses, axis<span style="color: #666666">=0</span>) <span style="color: #666666">/</span> np<span style="color: #666666">.</span>arange(<span style="color: #666666">1</span>, <span style="color: #666666">10001</span>)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">8</span>,<span style="color: #666666">3.5</span>))
|
||||
plt<span style="color: #666666">.</span>plot(cumulative_heads_ratio)
|
||||
plt<span style="color: #666666">.</span>plot([<span style="color: #666666">0</span>, <span style="color: #666666">10000</span>], [<span style="color: #666666">0.51</span>, <span style="color: #666666">0.51</span>], <span style="color: #BA2121">"k--"</span>, linewidth<span style="color: #666666">=2</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"51%"</span>)
|
||||
plt<span style="color: #666666">.</span>plot([<span style="color: #666666">0</span>, <span style="color: #666666">10000</span>], [<span style="color: #666666">0.5</span>, <span style="color: #666666">0.5</span>], <span style="color: #BA2121">"k-"</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"50%"</span>)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"Number of coin tosses"</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"Heads ratio"</span>)
|
||||
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">"lower right"</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>, <span style="color: #666666">10000</span>, <span style="color: #666666">0.42</span>, <span style="color: #666666">0.58</span>])
|
||||
save_fig(<span style="color: #BA2121">"votingsimple"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec5">Using the Voting Classifier </h2>
|
||||
|
||||
<p>
|
||||
We can use the voting classifier on other data sets, here the excting binary case of two distinct objects using the make moons functionality of -Scikit-Learn-.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_moons
|
||||
|
||||
X, y <span style="color: #666666">=</span> make_moons(n_samples<span style="color: #666666">=500</span>, noise<span style="color: #666666">=0.30</span>, random_state<span style="color: #666666">=42</span>)
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(X, y, random_state<span style="color: #666666">=42</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> VotingClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
||||
|
||||
log_clf <span style="color: #666666">=</span> LogisticRegression(solver<span style="color: #666666">=</span><span style="color: #BA2121">"liblinear"</span>, random_state<span style="color: #666666">=42</span>)
|
||||
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=10</span>, random_state<span style="color: #666666">=42</span>)
|
||||
svm_clf <span style="color: #666666">=</span> SVC(gamma<span style="color: #666666">=</span><span style="color: #BA2121">"auto"</span>, random_state<span style="color: #666666">=42</span>)
|
||||
|
||||
voting_clf <span style="color: #666666">=</span> VotingClassifier(
|
||||
estimators<span style="color: #666666">=</span>[(<span style="color: #BA2121">'lr'</span>, log_clf), (<span style="color: #BA2121">'rf'</span>, rnd_clf), (<span style="color: #BA2121">'svc'</span>, svm_clf)],
|
||||
voting<span style="color: #666666">=</span><span style="color: #BA2121">'hard'</span>)
|
||||
|
||||
voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> clf <span style="color: #AA22FF; font-weight: bold">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000">print</span>(clf<span style="color: #666666">.</span><span style="color: #19177C">__class__</span><span style="color: #666666">.</span><span style="color: #19177C">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
|
||||
log_clf <span style="color: #666666">=</span> LogisticRegression(solver<span style="color: #666666">=</span><span style="color: #BA2121">"liblinear"</span>, random_state<span style="color: #666666">=42</span>)
|
||||
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=10</span>, random_state<span style="color: #666666">=42</span>)
|
||||
svm_clf <span style="color: #666666">=</span> SVC(gamma<span style="color: #666666">=</span><span style="color: #BA2121">"auto"</span>, probability<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, random_state<span style="color: #666666">=42</span>)
|
||||
voting_clf <span style="color: #666666">=</span> VotingClassifier(
|
||||
estimators<span style="color: #666666">=</span>[(<span style="color: #BA2121">'lr'</span>, log_clf), (<span style="color: #BA2121">'rf'</span>, rnd_clf), (<span style="color: #BA2121">'svc'</span>, svm_clf)],
|
||||
voting<span style="color: #666666">=</span><span style="color: #BA2121">'soft'</span>)
|
||||
voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> clf <span style="color: #AA22FF; font-weight: bold">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000">print</span>(clf<span style="color: #666666">.</span><span style="color: #19177C">__class__</span><span style="color: #666666">.</span><span style="color: #19177C">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec6">Please, not the moons again! Voting and Bagging </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_moons
|
||||
|
||||
X, y <span style="color: #666666">=</span> make_moons(n_samples<span style="color: #666666">=500</span>, noise<span style="color: #666666">=0.30</span>, random_state<span style="color: #666666">=42</span>)
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(X, y, random_state<span style="color: #666666">=42</span>)
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> VotingClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
||||
|
||||
log_clf <span style="color: #666666">=</span> LogisticRegression(random_state<span style="color: #666666">=42</span>)
|
||||
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(random_state<span style="color: #666666">=42</span>)
|
||||
svm_clf <span style="color: #666666">=</span> SVC(random_state<span style="color: #666666">=42</span>)
|
||||
|
||||
voting_clf <span style="color: #666666">=</span> VotingClassifier(
|
||||
estimators<span style="color: #666666">=</span>[(<span style="color: #BA2121">'lr'</span>, log_clf), (<span style="color: #BA2121">'rf'</span>, rnd_clf), (<span style="color: #BA2121">'svc'</span>, svm_clf)],
|
||||
voting<span style="color: #666666">=</span><span style="color: #BA2121">'hard'</span>)
|
||||
voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> clf <span style="color: #AA22FF; font-weight: bold">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000">print</span>(clf<span style="color: #666666">.</span><span style="color: #19177C">__class__</span><span style="color: #666666">.</span><span style="color: #19177C">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>log_clf <span style="color: #666666">=</span> LogisticRegression(random_state<span style="color: #666666">=42</span>)
|
||||
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(random_state<span style="color: #666666">=42</span>)
|
||||
svm_clf <span style="color: #666666">=</span> SVC(probability<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, random_state<span style="color: #666666">=42</span>)
|
||||
|
||||
voting_clf <span style="color: #666666">=</span> VotingClassifier(
|
||||
estimators<span style="color: #666666">=</span>[(<span style="color: #BA2121">'lr'</span>, log_clf), (<span style="color: #BA2121">'rf'</span>, rnd_clf), (<span style="color: #BA2121">'svc'</span>, svm_clf)],
|
||||
voting<span style="color: #666666">=</span><span style="color: #BA2121">'soft'</span>)
|
||||
voting_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> clf <span style="color: #AA22FF; font-weight: bold">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000">print</span>(clf<span style="color: #666666">.</span><span style="color: #19177C">__class__</span><span style="color: #666666">.</span><span style="color: #19177C">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec7">Random forests </h2>
|
||||
|
||||
<p>
|
||||
Random forests provide an improvement over bagged trees by way of a
|
||||
@@ -210,7 +397,7 @@ this setting.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec3">Random Forest Algorithm </h2>
|
||||
<h2 id="___sec8">Random Forest Algorithm </h2>
|
||||
The algorithm described here can be applied to both classification and regression problems.
|
||||
|
||||
<p>
|
||||
@@ -236,7 +423,7 @@ We will grow of forest of say \( B \) trees.
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec4">Random Forests Compared with other Methods on the Cancer Data </h2>
|
||||
<h2 id="___sec9">Random Forests Compared with other Methods on the Cancer Data </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
@@ -309,7 +496,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec5">Compare Bagging on Trees with Random Forests </h2>
|
||||
<h2 id="___sec10">Compare Bagging on Trees with Random Forests </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
@@ -331,7 +518,7 @@ np<span style="color: #666666">.</span>sum(y_pred <span style="color: #666666">=
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec6">Boosting, a Bird's Eye View </h2>
|
||||
<h2 id="___sec11">Boosting, a Bird's Eye View </h2>
|
||||
|
||||
<p>
|
||||
The basic idea is to combine weak classifiers in order to create a good
|
||||
@@ -348,7 +535,7 @@ them with a factor.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec7">What is boosting? Additive Modelling/Iterative Fitting </h2>
|
||||
<h2 id="___sec12">What is boosting? Additive Modelling/Iterative Fitting </h2>
|
||||
|
||||
<p>
|
||||
Boosting is a way of fitting an additive expansion in a set of
|
||||
@@ -400,7 +587,7 @@ In iterative fitting or additive modeling, we minimize the cost function with re
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec8">Iterative Fitting, Regression and Squared-error Cost Function </h2>
|
||||
<h2 id="___sec13">Iterative Fitting, Regression and Squared-error Cost Function </h2>
|
||||
|
||||
<p>
|
||||
The way we proceed is as follows (here we specialize to the squared-error cost function)
|
||||
@@ -425,7 +612,7 @@ at the internal nodes, and the predictions at the terminal nodes.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec9">Squared-Error Example and Iterative Fitting </h2>
|
||||
<h2 id="___sec14">Squared-Error Example and Iterative Fitting </h2>
|
||||
|
||||
<p>
|
||||
To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function.
|
||||
@@ -473,7 +660,7 @@ The solution to these two equations gives us in turn \( \beta_1 \) and \( \gamma
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec10">Iterative Fitting, Classification and AdaBoost </h2>
|
||||
<h2 id="___sec15">Iterative Fitting, Classification and AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
Let us consider a binary classification problem with two outcomes \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
|
||||
@@ -508,7 +695,7 @@ $$
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec11">Adaptive Boosting, AdaBoost </h2>
|
||||
<h2 id="___sec16">Adaptive Boosting, AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
In our iterative procedure we define thus
|
||||
@@ -536,7 +723,7 @@ where we have defined \( w_i^m= \exp{(-y_if_{m-1}(x_i))} \).
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec12">Building up AdaBoost </h2>
|
||||
<h2 id="___sec17">Building up AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
First, for any \( \beta > 0 \), we optimize \( G \) by setting
|
||||
@@ -580,7 +767,7 @@ $$
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec13">Adaptive boosting: AdaBoost, Basic Algorithm </h2>
|
||||
<h2 id="___sec18">Adaptive boosting: AdaBoost, Basic Algorithm </h2>
|
||||
|
||||
<p>
|
||||
The algorithm here is rather straightforward. Assume that our weak
|
||||
@@ -602,7 +789,7 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec14">Basic Steps of AdaBoost </h2>
|
||||
<h2 id="___sec19">Basic Steps of AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
With the above definitions we are now ready to set up the algorithm for AdaBoost.
|
||||
@@ -642,7 +829,7 @@ observations that are missed in the previous iterations.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec15">AdaBoost Examples </h2>
|
||||
<h2 id="___sec20">AdaBoost Examples </h2>
|
||||
|
||||
<p>
|
||||
Using <b>Scikit-Learn</b> it is easy to apply the adaptive boosting algorithm, as done here.
|
||||
@@ -675,7 +862,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec16">AdaBoost for Regression </h2>
|
||||
<h2 id="___sec21">AdaBoost for Regression </h2>
|
||||
|
||||
<p>
|
||||
Here we present <a href="https://pdfs.semanticscholar.org/8d49/e2dedb817f2c3330e74b63c5fc86d2399ce3.pdf" target="_blank">Drucker's AdaBoost</a> tailored for regression.
|
||||
@@ -703,7 +890,7 @@ Start by selecting a set of training data \( n \) and assign to each entry a wei
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec17">Gradient boosting: Basics with Steepest Descent </h2>
|
||||
<h2 id="___sec22">Gradient boosting: Basics with Steepest Descent </h2>
|
||||
|
||||
<p>
|
||||
Gradient boosting is again a similar technique to Adaptive boosting,
|
||||
@@ -718,7 +905,7 @@ function was the least squares function.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec18">The Squared-Error again! Steepest Descent </h2>
|
||||
<h2 id="___sec23">The Squared-Error again! Steepest Descent </h2>
|
||||
|
||||
<p>
|
||||
We start again with our cost function \( {\cal C}(\boldsymbol{y}m\boldsymbol{f})=\sum_{i=0}^{n-1}{\cal L}(y_i, f(x_i)) \) where we want to minimize
|
||||
@@ -753,7 +940,7 @@ $$
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec19">Steepest Descent Example </h2>
|
||||
<h2 id="___sec24">Steepest Descent Example </h2>
|
||||
|
||||
<p>
|
||||
Optimizing with respect to \( \rho \) we obtain (taking the derivative) that \( \rho_1 = -1/2 \). We have then that
|
||||
@@ -771,7 +958,7 @@ and find a new value for \( \rho_2=-1/2 \) and continue till we have reached \(
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec20">Gradient Boosting, algorithm </h2>
|
||||
<h2 id="___sec25">Gradient Boosting, algorithm </h2>
|
||||
|
||||
<p>
|
||||
Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard squared-error function
|
||||
@@ -797,7 +984,7 @@ The way we proceed in an iterative fashion is to
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec21">Gradient Boosting Example, Regression </h2>
|
||||
<h2 id="___sec26">Gradient Boosting Example, Regression </h2>
|
||||
|
||||
<p>
|
||||
We discuss here the difference between the steepest descent approach and gradient boosting by repeating our simple regression example above.
|
||||
@@ -805,7 +992,7 @@ We discuss here the difference between the steepest descent approach and gradien
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec22">Gradient Boosting, Examples of Regression </h2>
|
||||
<h2 id="___sec27">Gradient Boosting, Examples of Regression </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
@@ -859,7 +1046,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec23">Gradient Boosting, Classification Example </h2>
|
||||
<h2 id="___sec28">Gradient Boosting, Classification Example </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
@@ -907,7 +1094,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec24">XGBoost: Extreme Gradient Boosting </h2>
|
||||
<h2 id="___sec29">XGBoost: Extreme Gradient Boosting </h2>
|
||||
|
||||
<p>
|
||||
<a href="https://github.com/dmlc/xgboost" target="_blank">XGBoost</a> or Extreme Gradient
|
||||
@@ -928,7 +1115,7 @@ It is now the algorithm which wins essentially all ML competitions!!!
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec25">Regression Case </h2>
|
||||
<h2 id="___sec30">Regression Case </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -983,7 +1170,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec26">Xgboost on the Cancer Data </h2>
|
||||
<h2 id="___sec31">Xgboost on the Cancer Data </h2>
|
||||
|
||||
<p>
|
||||
As you will see from the confusion matrix below, XGBoots does an excellent job on the Wisconsin cancer data and outperforms essentially all agorithms we have discussed till now.
|
||||
|
||||
Binary file not shown.
@@ -10,7 +10,7 @@
|
||||
"<!-- Author: --> \n",
|
||||
"**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n",
|
||||
"\n",
|
||||
"Date: **Oct 31, 2020**\n",
|
||||
"Date: **Nov 2, 2020**\n",
|
||||
"\n",
|
||||
"Copyright 1999-2020, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license\n",
|
||||
"\n",
|
||||
@@ -18,18 +18,226 @@
|
||||
"\n",
|
||||
"## Overview of week 45\n",
|
||||
"\n",
|
||||
"* \"Thursday: Wrapping up from last week. Bagging and Random forests.\n",
|
||||
"* \"Thursday: Wrapping up from last week. Bagging and Random forests. Boosting methods.\n",
|
||||
"\n",
|
||||
"* \"Friday: Boosting and gradient boosting\n",
|
||||
"\n",
|
||||
"Geron's chapter 7. See also lecture from [STK-IN4300, lecture 7](https://www.uio.no/studier/emner/matnat/math/STK-IN4300/h20/slides/lecture_7.pdf). Chapter 9.2 of Hastie et al contains also a good discussion.\n",
|
||||
"Geron's chapter 7. See also lecture from [STK-IN4300, lecture 9](https://www.uio.no/studier/emner/matnat/math/STK-IN4300/h20/slides/lecture_9.pdf). Chapter 10 (sections 10.1-10.10 are the most relevant ones) of Hastie et al contains also a good discussion.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## Thursday\n",
|
||||
"\n",
|
||||
"Bagging, voting and random forests.\n",
|
||||
"The material on bagging and voting is a repeat from last week and can be found in the slides from week 44.\n",
|
||||
"We repeat here the voting approach since this will serve as a motivation for boosting methods later.\n",
|
||||
"\n",
|
||||
"## Why Voting?\n",
|
||||
"\n",
|
||||
"The idea behind boosting, and voting as well can be phrased as follows:\n",
|
||||
"**Can a group of people somehow arrive at highly\n",
|
||||
"reasoned decisions, despite the weak judgement of the individual\n",
|
||||
"members?**\n",
|
||||
"\n",
|
||||
"The aim is to create a good classifier by combining several weak classifiers.\n",
|
||||
"**A weak classifier is a classifier which is able to produce results that are only slightly better than guessing at random.**\n",
|
||||
"\n",
|
||||
"The basic approach is to apply repeatedly (in boosting this is done in an iterative way) a weak classifier to modifications of the data.\n",
|
||||
"In voting we simply apply the law of large numbers while in boosting we give more weight to misclassified data in\n",
|
||||
"each iteration. \n",
|
||||
"\n",
|
||||
"Decision trees play an important role as our weak classifier. They serve as the basic method. \n",
|
||||
"\n",
|
||||
"## Tossing coins\n",
|
||||
"The simplest case is a so-called voting ensemble. To illustrate this, Think of you tossing coins with a biased outcome of 51 per cent for heads and 49% for tails.\n",
|
||||
"With only few tosses, you may not clearly see this distribution. However, after some thousands of tosses (sounds like you may have some spare time problems), there will be a clear majority of heads.\n",
|
||||
"With 2000 tosses you should see approximately 1020 heads and 980 tails.\n",
|
||||
"\n",
|
||||
"We can then state that the outcome is a clear majority of heads. If you do this ten thousand times, it is easy to see that there is a 97% likelihood of a majority of heads.\n",
|
||||
"\n",
|
||||
"Another example would be to collect all polls before an\n",
|
||||
"election. Different polls may show different likelihoods for a\n",
|
||||
"candidate winning with say a majority of the popular vote. The majority vote\n",
|
||||
"would then consist in many polls indicating that this candidate will\n",
|
||||
"actually win.\n",
|
||||
"\n",
|
||||
"The example here shows how we can implement the coin tossing case, clealry demostrating that after some tosses we see the law of large numbers kicking in.\n",
|
||||
"\n",
|
||||
"## Simple Voting Example, head or tail"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"heads_proba = 0.51\n",
|
||||
"coin_tosses = (np.random.rand(10000, 10) < heads_proba).astype(np.int32)\n",
|
||||
"cumulative_heads_ratio = np.cumsum(coin_tosses, axis=0) / np.arange(1, 10001).reshape(-1, 1)\n",
|
||||
"plt.figure(figsize=(8,3.5))\n",
|
||||
"plt.plot(cumulative_heads_ratio)\n",
|
||||
"plt.plot([0, 10000], [0.51, 0.51], \"k--\", linewidth=2, label=\"51%\")\n",
|
||||
"plt.plot([0, 10000], [0.5, 0.5], \"k-\", label=\"50%\")\n",
|
||||
"plt.xlabel(\"Number of coin tosses\")\n",
|
||||
"plt.ylabel(\"Heads ratio\")\n",
|
||||
"plt.legend(loc=\"lower right\")\n",
|
||||
"plt.axis([0, 10000, 0.42, 0.58])\n",
|
||||
"save_fig(\"votingsimple\")\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Using the Voting Classifier\n",
|
||||
"\n",
|
||||
"We can use the voting classifier on other data sets, here the excting binary case of two distinct objects using the make moons functionality of -Scikit-Learn-."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.datasets import make_moons\n",
|
||||
"\n",
|
||||
"X, y = make_moons(n_samples=500, noise=0.30, random_state=42)\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)\n",
|
||||
"\n",
|
||||
"from sklearn.ensemble import RandomForestClassifier\n",
|
||||
"from sklearn.ensemble import VotingClassifier\n",
|
||||
"from sklearn.linear_model import LogisticRegression\n",
|
||||
"from sklearn.svm import SVC\n",
|
||||
"\n",
|
||||
"log_clf = LogisticRegression(solver=\"liblinear\", random_state=42)\n",
|
||||
"rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)\n",
|
||||
"svm_clf = SVC(gamma=\"auto\", random_state=42)\n",
|
||||
"\n",
|
||||
"voting_clf = VotingClassifier(\n",
|
||||
" estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n",
|
||||
" voting='hard')\n",
|
||||
"\n",
|
||||
"voting_clf.fit(X_train, y_train)\n",
|
||||
"\n",
|
||||
"from sklearn.metrics import accuracy_score\n",
|
||||
"\n",
|
||||
"for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n",
|
||||
" clf.fit(X_train, y_train)\n",
|
||||
" y_pred = clf.predict(X_test)\n",
|
||||
" print(clf.__class__.__name__, accuracy_score(y_test, y_pred))\n",
|
||||
"\n",
|
||||
"log_clf = LogisticRegression(solver=\"liblinear\", random_state=42)\n",
|
||||
"rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)\n",
|
||||
"svm_clf = SVC(gamma=\"auto\", probability=True, random_state=42)\n",
|
||||
"voting_clf = VotingClassifier(\n",
|
||||
" estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n",
|
||||
" voting='soft')\n",
|
||||
"voting_clf.fit(X_train, y_train)\n",
|
||||
"\n",
|
||||
"from sklearn.metrics import accuracy_score\n",
|
||||
"\n",
|
||||
"for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n",
|
||||
" clf.fit(X_train, y_train)\n",
|
||||
" y_pred = clf.predict(X_test)\n",
|
||||
" print(clf.__class__.__name__, accuracy_score(y_test, y_pred))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Please, not the moons again! Voting and Bagging"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.datasets import make_moons\n",
|
||||
"\n",
|
||||
"X, y = make_moons(n_samples=500, noise=0.30, random_state=42)\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)\n",
|
||||
"from sklearn.ensemble import RandomForestClassifier\n",
|
||||
"from sklearn.ensemble import VotingClassifier\n",
|
||||
"from sklearn.linear_model import LogisticRegression\n",
|
||||
"from sklearn.svm import SVC\n",
|
||||
"\n",
|
||||
"log_clf = LogisticRegression(random_state=42)\n",
|
||||
"rnd_clf = RandomForestClassifier(random_state=42)\n",
|
||||
"svm_clf = SVC(random_state=42)\n",
|
||||
"\n",
|
||||
"voting_clf = VotingClassifier(\n",
|
||||
" estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n",
|
||||
" voting='hard')\n",
|
||||
"voting_clf.fit(X_train, y_train)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.metrics import accuracy_score\n",
|
||||
"\n",
|
||||
"for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n",
|
||||
" clf.fit(X_train, y_train)\n",
|
||||
" y_pred = clf.predict(X_test)\n",
|
||||
" print(clf.__class__.__name__, accuracy_score(y_test, y_pred))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"log_clf = LogisticRegression(random_state=42)\n",
|
||||
"rnd_clf = RandomForestClassifier(random_state=42)\n",
|
||||
"svm_clf = SVC(probability=True, random_state=42)\n",
|
||||
"\n",
|
||||
"voting_clf = VotingClassifier(\n",
|
||||
" estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n",
|
||||
" voting='soft')\n",
|
||||
"voting_clf.fit(X_train, y_train)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.metrics import accuracy_score\n",
|
||||
"\n",
|
||||
"for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n",
|
||||
" clf.fit(X_train, y_train)\n",
|
||||
" y_pred = clf.predict(X_test)\n",
|
||||
" print(clf.__class__.__name__, accuracy_score(y_test, y_pred))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Random forests\n",
|
||||
"\n",
|
||||
"Random forests provide an improvement over bagged trees by way of a\n",
|
||||
@@ -102,7 +310,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": 7,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -186,7 +394,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": 8,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -199,7 +407,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": 9,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -743,7 +951,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": 10,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -956,7 +1164,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": 11,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1019,7 +1227,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": 12,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1092,7 +1300,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": 13,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1157,7 +1365,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 14,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user