added code
This commit is contained in:
Morten Hjorth-Jensen
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -207,7 +212,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._week45-bs008.html">9</a></li>
|
||||
<li><a href="._week45-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -207,7 +212,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._week45-bs009.html">10</a></li>
|
||||
<li><a href="._week45-bs010.html">11</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs002.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,19 +169,120 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0002"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="boosting-a-bird-s-eye-view" class="anchor">Boosting, a Bird's Eye View </h2>
|
||||
<h2 id="brief-code-reminder-from-last-wekk" class="anchor">Brief code reminder from last wekk </h2>
|
||||
|
||||
<p>The basic idea is to combine weak classifiers in order to create a good
|
||||
classifier. With a weak classifier we often intend a classifier which
|
||||
produces results which are only slightly better than we would get by
|
||||
random guesses.
|
||||
</p>
|
||||
|
||||
<p>This is done by applying in an iterative way a weak (or a standard
|
||||
classifier like decision trees) to modify the data. In each iteration
|
||||
we emphasize those observations which are misclassified by weighting
|
||||
them with a factor.
|
||||
</p>
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># Common imports</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> Image
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">pydot</span> <span style="color: #008000; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeRegressor
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> export_graphviz
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler, OneHotEncoder
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.compose</span> <span style="color: #008000; font-weight: bold">import</span> ColumnTransformer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> Image
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">pydot</span> <span style="color: #008000; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> BaggingClassifier
|
||||
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Where to save the figures and data files</span>
|
||||
PROJECT_ROOT_DIR <span style="color: #666666">=</span> <span style="color: #BA2121">"Results"</span>
|
||||
FIGURE_ID <span style="color: #666666">=</span> <span style="color: #BA2121">"Results/FigureFiles"</span>
|
||||
DATA_ID <span style="color: #666666">=</span> <span style="color: #BA2121">"DataFiles/"</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(PROJECT_ROOT_DIR):
|
||||
os<span style="color: #666666">.</span>mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(FIGURE_ID):
|
||||
os<span style="color: #666666">.</span>makedirs(FIGURE_ID)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(DATA_ID):
|
||||
os<span style="color: #666666">.</span>makedirs(DATA_ID)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">image_path</span>(fig_id):
|
||||
<span style="color: #008000; font-weight: bold">return</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(FIGURE_ID, fig_id)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">data_path</span>(dat_id):
|
||||
<span style="color: #008000; font-weight: bold">return</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(DATA_ID, dat_id)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">save_fig</span>(fig_id):
|
||||
plt<span style="color: #666666">.</span>savefig(image_path(fig_id) <span style="color: #666666">+</span> <span style="color: #BA2121">".png"</span>, <span style="color: #008000">format</span><span style="color: #666666">=</span><span style="color: #BA2121">'png'</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Load the cancer data</span>
|
||||
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
||||
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
||||
<span style="color: #008000">print</span>(X_train<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #008000">print</span>(X_test<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic">#Scale the data</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
scaler<span style="color: #666666">.</span>fit(X_train)
|
||||
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
||||
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
||||
<span style="color: #408080; font-style: italic">#define methods</span>
|
||||
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
||||
logreg<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy Logistic Regression with scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Support Vector Machine</span>
|
||||
svm<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy SVM with scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Decision Trees</span>
|
||||
deep_tree_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Decision Trees and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(deep_tree_clf<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> LabelEncoder
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
|
||||
<span style="color: #408080; font-style: italic"># Data set not specificied</span>
|
||||
<span style="color: #408080; font-style: italic">#Instantiate the model with 500 trees and entropy as splitting criteria</span>
|
||||
Random_Forest_model <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=500</span>,criterion<span style="color: #666666">=</span><span style="color: #BA2121">"entropy"</span>)
|
||||
Random_Forest_model<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #408080; font-style: italic">#Cross validation</span>
|
||||
accuracy <span style="color: #666666">=</span> cross_validate(Random_Forest_model,X_test_scaled,y_test,cv<span style="color: #666666">=10</span>)[<span style="color: #BA2121">'test_score'</span>]
|
||||
<span style="color: #008000">print</span>(accuracy)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Random Forests and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(Random_Forest_model<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
y_pred <span style="color: #666666">=</span> Random_Forest_model<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
y_probas <span style="color: #666666">=</span> Random_Forest_model<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -195,7 +301,7 @@ them with a factor.
|
||||
<li><a href="._week45-bs010.html">11</a></li>
|
||||
<li><a href="._week45-bs011.html">12</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs003.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,53 +169,20 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0003"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="what-is-boosting-additive-modelling-iterative-fitting" class="anchor">What is boosting? Additive Modelling/Iterative Fitting </h2>
|
||||
<h2 id="boosting-a-bird-s-eye-view" class="anchor">Boosting, a Bird's Eye View </h2>
|
||||
|
||||
<p>Boosting is a way of fitting an additive expansion in a set of
|
||||
elementary basis functions like for example some simple polynomials.
|
||||
Assume for example that we have a function
|
||||
</p>
|
||||
$$
|
||||
f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m),
|
||||
$$
|
||||
|
||||
<p>where \( \beta_m \) are the expansion parameters to be determined in a
|
||||
minimization process and \( b(x;\gamma_m) \) are some simple functions of
|
||||
the multivariable parameter \( x \) which is characterized by the
|
||||
parameters \( \gamma_m \).
|
||||
<p>The basic idea is to combine weak classifiers in order to create a good
|
||||
classifier. With a weak classifier we often intend a classifier which
|
||||
produces results which are only slightly better than we would get by
|
||||
random guesses.
|
||||
</p>
|
||||
|
||||
<p>As an example, consider the Sigmoid function we used in logistic
|
||||
regression. In that case, we can translate the function
|
||||
\( b(x;\gamma_m) \) into the Sigmoid function
|
||||
<p>This is done by applying in an iterative way a weak (or a standard
|
||||
classifier like decision trees) to modify the data. In each iteration
|
||||
we emphasize those observations which are misclassified by weighting
|
||||
them with a factor.
|
||||
</p>
|
||||
|
||||
$$
|
||||
\sigma(t) = \frac{1}{1+\exp{(-t)}},
|
||||
$$
|
||||
|
||||
<p>where \( t=\gamma_0+\gamma_1 x \) and the parameters \( \gamma_0 \) and
|
||||
\( \gamma_1 \) were determined by the Logistic Regression fitting
|
||||
algorithm.
|
||||
</p>
|
||||
|
||||
<p>As another example, consider the cost function we defined for linear regression</p>
|
||||
$$
|
||||
C(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f(x_i))^2.
|
||||
$$
|
||||
|
||||
<p>In this case the function \( f(x) \) was replaced by the design matrix
|
||||
\( \boldsymbol{X} \) and the unknown linear regression parameters \( \boldsymbol{\beta} \),
|
||||
that is \( \boldsymbol{f}=\boldsymbol{X}\boldsymbol{\beta} \). In linear regression we can
|
||||
simply invert a matrix and obtain the parameters \( \beta \) by
|
||||
</p>
|
||||
|
||||
$$
|
||||
\boldsymbol{\beta}=\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
|
||||
$$
|
||||
|
||||
<p>In iterative fitting or additive modeling, we minimize the cost function with respect to the parameters \( \beta_m \) and \( \gamma_m \).</p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
@@ -229,7 +201,7 @@ $$
|
||||
<li><a href="._week45-bs011.html">12</a></li>
|
||||
<li><a href="._week45-bs012.html">13</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs004.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,24 +169,52 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0004"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="iterative-fitting-regression-and-squared-error-cost-function" class="anchor">Iterative Fitting, Regression and Squared-error Cost Function </h2>
|
||||
<h2 id="what-is-boosting-additive-modelling-iterative-fitting" class="anchor">What is boosting? Additive Modelling/Iterative Fitting </h2>
|
||||
|
||||
<p>The way we proceed is as follows (here we specialize to the squared-error cost function)</p>
|
||||
|
||||
<ol>
|
||||
<li> Establish a cost function, here \( {\cal C}(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f_M(x_i))^2 \) with \( f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m) \).</li>
|
||||
<li> Initialize with a guess \( f_0(x) \). It could be one or even zero or some random numbers.</li>
|
||||
<li> For \( m=1:M \)
|
||||
<ol type="a"></li>
|
||||
<li> minimize \( \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2 \) wrt \( \gamma \) and \( \beta \)</li>
|
||||
<li> This gives the optimal values \( \beta_m \) and \( \gamma_m \)</li>
|
||||
<li> Determine then the new values \( f_m(x)=f_{m-1}(x) +\beta_m b(x;\gamma_m) \)</li>
|
||||
</ol>
|
||||
</ol>
|
||||
<p>We could use any of the algorithms we have discussed till now. If we
|
||||
use trees, \( \gamma \) parameterizes the split variables and split points
|
||||
at the internal nodes, and the predictions at the terminal nodes.
|
||||
<p>Boosting is a way of fitting an additive expansion in a set of
|
||||
elementary basis functions like for example some simple polynomials.
|
||||
Assume for example that we have a function
|
||||
</p>
|
||||
$$
|
||||
f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m),
|
||||
$$
|
||||
|
||||
<p>where \( \beta_m \) are the expansion parameters to be determined in a
|
||||
minimization process and \( b(x;\gamma_m) \) are some simple functions of
|
||||
the multivariable parameter \( x \) which is characterized by the
|
||||
parameters \( \gamma_m \).
|
||||
</p>
|
||||
|
||||
<p>As an example, consider the Sigmoid function we used in logistic
|
||||
regression. In that case, we can translate the function
|
||||
\( b(x;\gamma_m) \) into the Sigmoid function
|
||||
</p>
|
||||
|
||||
$$
|
||||
\sigma(t) = \frac{1}{1+\exp{(-t)}},
|
||||
$$
|
||||
|
||||
<p>where \( t=\gamma_0+\gamma_1 x \) and the parameters \( \gamma_0 \) and
|
||||
\( \gamma_1 \) were determined by the Logistic Regression fitting
|
||||
algorithm.
|
||||
</p>
|
||||
|
||||
<p>As another example, consider the cost function we defined for linear regression</p>
|
||||
$$
|
||||
C(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f(x_i))^2.
|
||||
$$
|
||||
|
||||
<p>In this case the function \( f(x) \) was replaced by the design matrix
|
||||
\( \boldsymbol{X} \) and the unknown linear regression parameters \( \boldsymbol{\beta} \),
|
||||
that is \( \boldsymbol{f}=\boldsymbol{X}\boldsymbol{\beta} \). In linear regression we can
|
||||
simply invert a matrix and obtain the parameters \( \beta \) by
|
||||
</p>
|
||||
|
||||
$$
|
||||
\boldsymbol{\beta}=\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
|
||||
$$
|
||||
|
||||
<p>In iterative fitting or additive modeling, we minimize the cost function with respect to the parameters \( \beta_m \) and \( \gamma_m \).</p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -202,7 +235,7 @@ at the internal nodes, and the predictions at the terminal nodes.
|
||||
<li><a href="._week45-bs012.html">13</a></li>
|
||||
<li><a href="._week45-bs013.html">14</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs005.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,46 +169,23 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0005"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="squared-error-example-and-iterative-fitting" class="anchor">Squared-Error Example and Iterative Fitting </h2>
|
||||
<h2 id="iterative-fitting-regression-and-squared-error-cost-function" class="anchor">Iterative Fitting, Regression and Squared-error Cost Function </h2>
|
||||
|
||||
<p>To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function.</p>
|
||||
<p>The way we proceed is as follows (here we specialize to the squared-error cost function)</p>
|
||||
|
||||
<p>For simplicity we assume also that our functions \( b(x;\gamma)=1+\gamma x \). </p>
|
||||
|
||||
<p>This means that for every iteration \( m \), we need to optimize</p>
|
||||
|
||||
$$
|
||||
(\beta_m,\gamma_m) = \mathrm{argmin}_{\beta,\lambda}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2=\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta(1+\gamma x_i))^2.
|
||||
$$
|
||||
|
||||
<p>We start our iteration by simply setting \( f_0(x)=0 \).
|
||||
Taking the derivatives with respect to \( \beta \) and \( \gamma \) we obtain
|
||||
</p>
|
||||
$$
|
||||
\frac{\partial {\cal C}}{\partial \beta} = -2\sum_{i}(1+\gamma x_i)(y_i-\beta(1+\gamma x_i))=0,
|
||||
$$
|
||||
|
||||
<p>and</p>
|
||||
$$
|
||||
\frac{\partial {\cal C}}{\partial \gamma} =-2\sum_{i}\beta x_i(y_i-\beta(1+\gamma x_i))=0.
|
||||
$$
|
||||
|
||||
<p>We can then rewrite these equations as (defining \( \boldsymbol{w}=\boldsymbol{e}+\gamma \boldsymbol{x}) \) with \( \boldsymbol{e} \) being the unit vector)</p>
|
||||
$$
|
||||
\gamma \boldsymbol{w}^T(\boldsymbol{y}-\beta\gamma \boldsymbol{w})=0,
|
||||
$$
|
||||
|
||||
<p>which gives us \( \beta = \boldsymbol{w}^T\boldsymbol{y}/(\boldsymbol{w}^T\boldsymbol{w}) \). Similarly we have </p>
|
||||
$$
|
||||
\beta\gamma \boldsymbol{x}^T(\boldsymbol{y}-\beta(1+\gamma \boldsymbol{x}))=0,
|
||||
$$
|
||||
|
||||
<p>which leads to \( \gamma =(\boldsymbol{x}^T\boldsymbol{y}-\beta\boldsymbol{x}^T\boldsymbol{e})/(\beta\boldsymbol{x}^T\boldsymbol{x}) \). Inserting
|
||||
for \( \beta \) gives us an equation for \( \gamma \). This is a non-linear equation in the unknown \( \gamma \) and has to be solved numerically.
|
||||
</p>
|
||||
|
||||
<p>The solution to these two equations gives us in turn \( \beta_1 \) and \( \gamma_1 \) leading to the new expression for \( f_1(x) \) as
|
||||
\( f_1(x) = \beta_1(1+\gamma_1x) \). Doing this \( M \) times results in our final estimate for the function \( f \).
|
||||
<ol>
|
||||
<li> Establish a cost function, here \( {\cal C}(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f_M(x_i))^2 \) with \( f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m) \).</li>
|
||||
<li> Initialize with a guess \( f_0(x) \). It could be one or even zero or some random numbers.</li>
|
||||
<li> For \( m=1:M \)
|
||||
<ol type="a"></li>
|
||||
<li> minimize \( \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2 \) wrt \( \gamma \) and \( \beta \)</li>
|
||||
<li> This gives the optimal values \( \beta_m \) and \( \gamma_m \)</li>
|
||||
<li> Determine then the new values \( f_m(x)=f_{m-1}(x) +\beta_m b(x;\gamma_m) \)</li>
|
||||
</ol>
|
||||
</ol>
|
||||
<p>We could use any of the algorithms we have discussed till now. If we
|
||||
use trees, \( \gamma \) parameterizes the split variables and split points
|
||||
at the internal nodes, and the predictions at the terminal nodes.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
@@ -226,7 +208,7 @@ for \( \beta \) gives us an equation for \( \gamma \). This is a non-linear equa
|
||||
<li><a href="._week45-bs013.html">14</a></li>
|
||||
<li><a href="._week45-bs014.html">15</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs006.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,37 +169,48 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0006"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="iterative-fitting-classification-and-adaboost" class="anchor">Iterative Fitting, Classification and AdaBoost </h2>
|
||||
<h2 id="squared-error-example-and-iterative-fitting" class="anchor">Squared-Error Example and Iterative Fitting </h2>
|
||||
|
||||
<p>Let us consider a binary classification problem with two outcomes \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
|
||||
observations. We define a classification function \( G(x) \) which produces a prediction taking one or the other of the two values
|
||||
\( \{-1,1\} \).
|
||||
<p>To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function.</p>
|
||||
|
||||
<p>For simplicity we assume also that our functions \( b(x;\gamma)=1+\gamma x \). </p>
|
||||
|
||||
<p>This means that for every iteration \( m \), we need to optimize</p>
|
||||
|
||||
$$
|
||||
(\beta_m,\gamma_m) = \mathrm{argmin}_{\beta,\lambda}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2=\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta(1+\gamma x_i))^2.
|
||||
$$
|
||||
|
||||
<p>We start our iteration by simply setting \( f_0(x)=0 \).
|
||||
Taking the derivatives with respect to \( \beta \) and \( \gamma \) we obtain
|
||||
</p>
|
||||
$$
|
||||
\frac{\partial {\cal C}}{\partial \beta} = -2\sum_{i}(1+\gamma x_i)(y_i-\beta(1+\gamma x_i))=0,
|
||||
$$
|
||||
|
||||
<p>and</p>
|
||||
$$
|
||||
\frac{\partial {\cal C}}{\partial \gamma} =-2\sum_{i}\beta x_i(y_i-\beta(1+\gamma x_i))=0.
|
||||
$$
|
||||
|
||||
<p>We can then rewrite these equations as (defining \( \boldsymbol{w}=\boldsymbol{e}+\gamma \boldsymbol{x}) \) with \( \boldsymbol{e} \) being the unit vector)</p>
|
||||
$$
|
||||
\gamma \boldsymbol{w}^T(\boldsymbol{y}-\beta\gamma \boldsymbol{w})=0,
|
||||
$$
|
||||
|
||||
<p>which gives us \( \beta = \boldsymbol{w}^T\boldsymbol{y}/(\boldsymbol{w}^T\boldsymbol{w}) \). Similarly we have </p>
|
||||
$$
|
||||
\beta\gamma \boldsymbol{x}^T(\boldsymbol{y}-\beta(1+\gamma \boldsymbol{x}))=0,
|
||||
$$
|
||||
|
||||
<p>which leads to \( \gamma =(\boldsymbol{x}^T\boldsymbol{y}-\beta\boldsymbol{x}^T\boldsymbol{e})/(\beta\boldsymbol{x}^T\boldsymbol{x}) \). Inserting
|
||||
for \( \beta \) gives us an equation for \( \gamma \). This is a non-linear equation in the unknown \( \gamma \) and has to be solved numerically.
|
||||
</p>
|
||||
|
||||
<p>The error rate of the training sample is then</p>
|
||||
|
||||
$$
|
||||
\mathrm{\overline{err}}=\frac{1}{n} \sum_{i=0}^{n-1} I(y_i\ne G(x_i)).
|
||||
$$
|
||||
|
||||
<p>The iterative procedure starts with defining a weak classifier whose
|
||||
error rate is barely better than random guessing. The iterative
|
||||
procedure in boosting is to sequentially apply a weak
|
||||
classification algorithm to repeatedly modified versions of the data
|
||||
producing a sequence of weak classifiers \( G_m(x) \).
|
||||
<p>The solution to these two equations gives us in turn \( \beta_1 \) and \( \gamma_1 \) leading to the new expression for \( f_1(x) \) as
|
||||
\( f_1(x) = \beta_1(1+\gamma_1x) \). Doing this \( M \) times results in our final estimate for the function \( f \).
|
||||
</p>
|
||||
|
||||
<p>Here we will express our function \( f(x) \) in terms of \( G(x) \). That is</p>
|
||||
$$
|
||||
f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m),
|
||||
$$
|
||||
|
||||
<p>will be a function of </p>
|
||||
$$
|
||||
G_M(x) = \mathrm{sign} \sum_{i=1}^M \alpha_m G_m(x).
|
||||
$$
|
||||
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
@@ -216,7 +232,7 @@ $$
|
||||
<li><a href="._week45-bs014.html">15</a></li>
|
||||
<li><a href="._week45-bs015.html">16</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs007.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,29 +169,36 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0007"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="adaptive-boosting-adaboost" class="anchor">Adaptive Boosting, AdaBoost </h2>
|
||||
<h2 id="iterative-fitting-classification-and-adaboost" class="anchor">Iterative Fitting, Classification and AdaBoost </h2>
|
||||
|
||||
<p>In our iterative procedure we define thus</p>
|
||||
$$
|
||||
f_m(x) = f_{m-1}(x)+\beta_mG_m(x).
|
||||
$$
|
||||
|
||||
<p>The simplest possible cost function which leads (also simple from a computational point of view) to the AdaBoost algorithm is the
|
||||
exponential cost/loss function defined as
|
||||
</p>
|
||||
$$
|
||||
C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}\exp{(-y_i(f_{m-1}(x_i)+\beta G(x_i))}.
|
||||
$$
|
||||
|
||||
<p>We optimize \( \beta \) and \( G \) for each value of \( m=1:M \) as we did in the regression case.
|
||||
This is normally done in two steps. Let us however first rewrite the cost function as
|
||||
<p>Let us consider a binary classification problem with two outcomes \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
|
||||
observations. We define a classification function \( G(x) \) which produces a prediction taking one or the other of the two values
|
||||
\( \{-1,1\} \).
|
||||
</p>
|
||||
|
||||
<p>The error rate of the training sample is then</p>
|
||||
|
||||
$$
|
||||
C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}w_i^{m}\exp{(-y_i\beta G(x_i))},
|
||||
\mathrm{\overline{err}}=\frac{1}{n} \sum_{i=0}^{n-1} I(y_i\ne G(x_i)).
|
||||
$$
|
||||
|
||||
<p>The iterative procedure starts with defining a weak classifier whose
|
||||
error rate is barely better than random guessing. The iterative
|
||||
procedure in boosting is to sequentially apply a weak
|
||||
classification algorithm to repeatedly modified versions of the data
|
||||
producing a sequence of weak classifiers \( G_m(x) \).
|
||||
</p>
|
||||
|
||||
<p>Here we will express our function \( f(x) \) in terms of \( G(x) \). That is</p>
|
||||
$$
|
||||
f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m),
|
||||
$$
|
||||
|
||||
<p>will be a function of </p>
|
||||
$$
|
||||
G_M(x) = \mathrm{sign} \sum_{i=1}^M \alpha_m G_m(x).
|
||||
$$
|
||||
|
||||
<p>where we have defined \( w_i^m= \exp{(-y_if_{m-1}(x_i))} \).</p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -210,7 +222,7 @@ $$
|
||||
<li><a href="._week45-bs015.html">16</a></li>
|
||||
<li><a href="._week45-bs016.html">17</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs008.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,45 +169,29 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0008"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="building-up-adaboost" class="anchor">Building up AdaBoost </h2>
|
||||
<h2 id="adaptive-boosting-adaboost" class="anchor">Adaptive Boosting, AdaBoost </h2>
|
||||
|
||||
<p>First, for any \( \beta > 0 \), we optimize \( G \) by setting</p>
|
||||
<p>In our iterative procedure we define thus</p>
|
||||
$$
|
||||
G_m(x) = \mathrm{sign} \sum_{i=0}^{n-1} w_i^m I(y_i \ne G_(x_i)),
|
||||
f_m(x) = f_{m-1}(x)+\beta_mG_m(x).
|
||||
$$
|
||||
|
||||
<p>which is the classifier that minimizes the weighted error rate in predicting \( y \).</p>
|
||||
|
||||
<p>We can do this by rewriting</p>
|
||||
<p>The simplest possible cost function which leads (also simple from a computational point of view) to the AdaBoost algorithm is the
|
||||
exponential cost/loss function defined as
|
||||
</p>
|
||||
$$
|
||||
\exp{-(\beta)}\sum_{y_i=G(x_i)}w_i^m+\exp{(\beta)}\sum_{y_i\ne G(x_i)}w_i^m,
|
||||
C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}\exp{(-y_i(f_{m-1}(x_i)+\beta G(x_i))}.
|
||||
$$
|
||||
|
||||
<p>which can be rewritten as</p>
|
||||
<p>We optimize \( \beta \) and \( G \) for each value of \( m=1:M \) as we did in the regression case.
|
||||
This is normally done in two steps. Let us however first rewrite the cost function as
|
||||
</p>
|
||||
|
||||
$$
|
||||
(\exp{(\beta)}-\exp{-(\beta)})\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i))+\exp{(-\beta)}\sum_{i=0}^{n-1}w_i^m=0,
|
||||
$$
|
||||
|
||||
<p>which leads to</p>
|
||||
$$
|
||||
\beta_m = \frac{1}{2}\log{\frac{1-\mathrm{\overline{err}}}{\mathrm{\overline{err}}}},
|
||||
$$
|
||||
|
||||
<p>where we have redefined the error as </p>
|
||||
$$
|
||||
\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i)}{\sum_{i=0}^{n-1}w_i^m},
|
||||
$$
|
||||
|
||||
<p>which leads to an update of</p>
|
||||
$$
|
||||
f_m(x) = f_{m-1}(x) +\beta_m G_m(x).
|
||||
$$
|
||||
|
||||
<p>This leads to the new weights</p>
|
||||
$$
|
||||
w_i^{m+1} = w_i^m \exp{(-y_i\beta_m G_m(x_i))}
|
||||
C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}w_i^{m}\exp{(-y_i\beta G(x_i))},
|
||||
$$
|
||||
|
||||
<p>where we have defined \( w_i^m= \exp{(-y_if_{m-1}(x_i))} \).</p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -227,7 +216,7 @@ $$
|
||||
<li><a href="._week45-bs016.html">17</a></li>
|
||||
<li><a href="._week45-bs017.html">18</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs009.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,23 +169,45 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0009"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="adaptive-boosting-adaboost-basic-algorithm" class="anchor">Adaptive boosting: AdaBoost, Basic Algorithm </h2>
|
||||
<h2 id="building-up-adaboost" class="anchor">Building up AdaBoost </h2>
|
||||
|
||||
<p>The algorithm here is rather straightforward. Assume that our weak
|
||||
classifier is a decision tree and we consider a binary set of outputs
|
||||
with \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
|
||||
observations. Our design matrix is given in terms of the
|
||||
feature/predictor vectors
|
||||
\( \boldsymbol{X}=[\boldsymbol{x}_0\boldsymbol{x}_1\dots\boldsymbol{x}_{p-1}] \). Finally, we define also a
|
||||
classifier determined by our data via a function \( G(x) \). This function tells us how well we are able to classify our outputs/targets \( \boldsymbol{y} \).
|
||||
</p>
|
||||
|
||||
<p>We have already defined the misclassification error \( \mathrm{err} \) as</p>
|
||||
<p>First, for any \( \beta > 0 \), we optimize \( G \) by setting</p>
|
||||
$$
|
||||
\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(x_i)),
|
||||
G_m(x) = \mathrm{sign} \sum_{i=0}^{n-1} w_i^m I(y_i \ne G_(x_i)),
|
||||
$$
|
||||
|
||||
<p>which is the classifier that minimizes the weighted error rate in predicting \( y \).</p>
|
||||
|
||||
<p>We can do this by rewriting</p>
|
||||
$$
|
||||
\exp{-(\beta)}\sum_{y_i=G(x_i)}w_i^m+\exp{(\beta)}\sum_{y_i\ne G(x_i)}w_i^m,
|
||||
$$
|
||||
|
||||
<p>which can be rewritten as</p>
|
||||
$$
|
||||
(\exp{(\beta)}-\exp{-(\beta)})\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i))+\exp{(-\beta)}\sum_{i=0}^{n-1}w_i^m=0,
|
||||
$$
|
||||
|
||||
<p>which leads to</p>
|
||||
$$
|
||||
\beta_m = \frac{1}{2}\log{\frac{1-\mathrm{\overline{err}}}{\mathrm{\overline{err}}}},
|
||||
$$
|
||||
|
||||
<p>where we have redefined the error as </p>
|
||||
$$
|
||||
\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i)}{\sum_{i=0}^{n-1}w_i^m},
|
||||
$$
|
||||
|
||||
<p>which leads to an update of</p>
|
||||
$$
|
||||
f_m(x) = f_{m-1}(x) +\beta_m G_m(x).
|
||||
$$
|
||||
|
||||
<p>This leads to the new weights</p>
|
||||
$$
|
||||
w_i^{m+1} = w_i^m \exp{(-y_i\beta_m G_m(x_i))}
|
||||
$$
|
||||
|
||||
<p>where the function \( I() \) is one if we misclassify and zero if we classify correctly. </p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -206,7 +233,7 @@ $$
|
||||
<li><a href="._week45-bs017.html">18</a></li>
|
||||
<li><a href="._week45-bs018.html">19</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs010.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,37 +169,23 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0010"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="basic-steps-of-adaboost" class="anchor">Basic Steps of AdaBoost </h2>
|
||||
<h2 id="adaptive-boosting-adaboost-basic-algorithm" class="anchor">Adaptive boosting: AdaBoost, Basic Algorithm </h2>
|
||||
|
||||
<p>With the above definitions we are now ready to set up the algorithm for AdaBoost.
|
||||
The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases.
|
||||
<p>The algorithm here is rather straightforward. Assume that our weak
|
||||
classifier is a decision tree and we consider a binary set of outputs
|
||||
with \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
|
||||
observations. Our design matrix is given in terms of the
|
||||
feature/predictor vectors
|
||||
\( \boldsymbol{X}=[\boldsymbol{x}_0\boldsymbol{x}_1\dots\boldsymbol{x}_{p-1}] \). Finally, we define also a
|
||||
classifier determined by our data via a function \( G(x) \). This function tells us how well we are able to classify our outputs/targets \( \boldsymbol{y} \).
|
||||
</p>
|
||||
<ol>
|
||||
<li> We start by initializing all weights to \( w_i = 1/n \), with \( i=0,1,2,\dots n-1 \). It is easy to see that we must have \( \sum_{i=0}^{n-1}w_i = 1 \).</li>
|
||||
<li> We rewrite the misclassification error as</li>
|
||||
</ol>
|
||||
|
||||
<p>We have already defined the misclassification error \( \mathrm{err} \) as</p>
|
||||
$$
|
||||
\mathrm{\overline{err}}_m=\frac{\sum_{i=0}^{n-1}w_i^m I(y_i\ne G(x_i))}{\sum_{i=0}^{n-1}w_i},
|
||||
\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(x_i)),
|
||||
$$
|
||||
|
||||
<ol>
|
||||
<li> Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree.
|
||||
<ol type="a"></li>
|
||||
<li> Fit then a given classifier to the training set using the weights \( w_i \).</li>
|
||||
<li> Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.</li>
|
||||
<li> Define a quantity \( \alpha_{m} = \log{(1-\mathrm{\overline{err}}_m)/\mathrm{\overline{err}}_m} \)</li>
|
||||
<li> Set the new weights to \( w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(x_i)} \).</li>
|
||||
</ol>
|
||||
<li> Compute the new classifier \( G(x)= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(x_i) \).</li>
|
||||
</ol>
|
||||
<p>For the iterations with \( m \le 2 \) the weights are modified
|
||||
individually at each steps. The observations which were misclassified
|
||||
at iteration \( m-1 \) have a weight which is larger than those which were
|
||||
classified properly. As this proceeds, the observations which were
|
||||
difficult to classifiy correctly are given a larger influence. Each
|
||||
new classification step \( m \) is then forced to concentrate on those
|
||||
observations that are missed in the previous iterations.
|
||||
</p>
|
||||
<p>where the function \( I() \) is one if we misclassify and zero if we classify correctly. </p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -221,7 +212,7 @@ observations that are missed in the previous iterations.
|
||||
<li><a href="._week45-bs018.html">19</a></li>
|
||||
<li><a href="._week45-bs019.html">20</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs011.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,53 +169,37 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0011"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="adaboost-examples" class="anchor">AdaBoost Examples </h2>
|
||||
<h2 id="basic-steps-of-adaboost" class="anchor">Basic Steps of AdaBoost </h2>
|
||||
|
||||
<p>Using <b>Scikit-Learn</b> it is easy to apply the adaptive boosting algorithm, as done here.</p>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> AdaBoostClassifier
|
||||
|
||||
ada_clf <span style="color: #666666">=</span> AdaBoostClassifier(
|
||||
DecisionTreeClassifier(max_depth<span style="color: #666666">=1</span>), n_estimators<span style="color: #666666">=200</span>,
|
||||
algorithm<span style="color: #666666">=</span><span style="color: #BA2121">"SAMME.R"</span>, learning_rate<span style="color: #666666">=0.5</span>, random_state<span style="color: #666666">=42</span>)
|
||||
ada_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> AdaBoostClassifier
|
||||
|
||||
ada_clf <span style="color: #666666">=</span> AdaBoostClassifier(
|
||||
DecisionTreeClassifier(max_depth<span style="color: #666666">=1</span>), n_estimators<span style="color: #666666">=200</span>,
|
||||
algorithm<span style="color: #666666">=</span><span style="color: #BA2121">"SAMME.R"</span>, learning_rate<span style="color: #666666">=0.5</span>, random_state<span style="color: #666666">=42</span>)
|
||||
ada_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
y_pred <span style="color: #666666">=</span> ada_clf<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
y_probas <span style="color: #666666">=</span> ada_clf<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<p>With the above definitions we are now ready to set up the algorithm for AdaBoost.
|
||||
The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases.
|
||||
</p>
|
||||
<ol>
|
||||
<li> We start by initializing all weights to \( w_i = 1/n \), with \( i=0,1,2,\dots n-1 \). It is easy to see that we must have \( \sum_{i=0}^{n-1}w_i = 1 \).</li>
|
||||
<li> We rewrite the misclassification error as</li>
|
||||
</ol>
|
||||
$$
|
||||
\mathrm{\overline{err}}_m=\frac{\sum_{i=0}^{n-1}w_i^m I(y_i\ne G(x_i))}{\sum_{i=0}^{n-1}w_i},
|
||||
$$
|
||||
|
||||
<ol>
|
||||
<li> Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree.
|
||||
<ol type="a"></li>
|
||||
<li> Fit then a given classifier to the training set using the weights \( w_i \).</li>
|
||||
<li> Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.</li>
|
||||
<li> Define a quantity \( \alpha_{m} = \log{(1-\mathrm{\overline{err}}_m)/\mathrm{\overline{err}}_m} \)</li>
|
||||
<li> Set the new weights to \( w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(x_i)} \).</li>
|
||||
</ol>
|
||||
<li> Compute the new classifier \( G(x)= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(x_i) \).</li>
|
||||
</ol>
|
||||
<p>For the iterations with \( m \le 2 \) the weights are modified
|
||||
individually at each steps. The observations which were misclassified
|
||||
at iteration \( m-1 \) have a weight which is larger than those which were
|
||||
classified properly. As this proceeds, the observations which were
|
||||
difficult to classifiy correctly are given a larger influence. Each
|
||||
new classification step \( m \) is then forced to concentrate on those
|
||||
observations that are missed in the previous iterations.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -236,6 +225,8 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._week45-bs018.html">19</a></li>
|
||||
<li><a href="._week45-bs019.html">20</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs012.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,17 +169,53 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0012"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" class="anchor">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent </h2>
|
||||
<h2 id="adaboost-examples" class="anchor">AdaBoost Examples </h2>
|
||||
|
||||
<p>Gradient boosting is again a similar technique to Adaptive boosting,
|
||||
it combines so-called weak classifiers or regressors into a strong
|
||||
method via a series of iterations.
|
||||
</p>
|
||||
<p>Using <b>Scikit-Learn</b> it is easy to apply the adaptive boosting algorithm, as done here.</p>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> AdaBoostClassifier
|
||||
|
||||
ada_clf <span style="color: #666666">=</span> AdaBoostClassifier(
|
||||
DecisionTreeClassifier(max_depth<span style="color: #666666">=1</span>), n_estimators<span style="color: #666666">=200</span>,
|
||||
algorithm<span style="color: #666666">=</span><span style="color: #BA2121">"SAMME.R"</span>, learning_rate<span style="color: #666666">=0.5</span>, random_state<span style="color: #666666">=42</span>)
|
||||
ada_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> AdaBoostClassifier
|
||||
|
||||
ada_clf <span style="color: #666666">=</span> AdaBoostClassifier(
|
||||
DecisionTreeClassifier(max_depth<span style="color: #666666">=1</span>), n_estimators<span style="color: #666666">=200</span>,
|
||||
algorithm<span style="color: #666666">=</span><span style="color: #BA2121">"SAMME.R"</span>, learning_rate<span style="color: #666666">=0.5</span>, random_state<span style="color: #666666">=42</span>)
|
||||
ada_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
y_pred <span style="color: #666666">=</span> ada_clf<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
y_probas <span style="color: #666666">=</span> ada_clf<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>In order to understand the method, let us illustrate its basics by
|
||||
bringing back the essential steps in linear regression, where our cost
|
||||
function was the least squares function.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -199,6 +240,7 @@ function was the least squares function.
|
||||
<li><a href="._week45-bs018.html">19</a></li>
|
||||
<li><a href="._week45-bs019.html">20</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs013.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,36 +169,18 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0013"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="the-squared-error-again-steepest-descent" class="anchor">The Squared-Error again! Steepest Descent </h2>
|
||||
<h2 id="gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" class="anchor">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent </h2>
|
||||
|
||||
<p>We start again with our cost function \( {\cal C}(\boldsymbol{y}m\boldsymbol{f})=\sum_{i=0}^{n-1}{\cal L}(y_i, f(x_i)) \) where we want to minimize
|
||||
This means that for every iteration, we need to optimize
|
||||
<p>Gradient boosting is again a similar technique to Adaptive boosting,
|
||||
it combines so-called weak classifiers or regressors into a strong
|
||||
method via a series of iterations.
|
||||
</p>
|
||||
|
||||
$$
|
||||
(\hat{\boldsymbol{f}}) = \mathrm{argmin}_{\boldsymbol{f}}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f(x_i))^2.
|
||||
$$
|
||||
|
||||
<p>We define a real function \( h_m(x) \) that defines our final function \( f_M(x) \) as</p>
|
||||
$$
|
||||
f_M(x) = \sum_{m=0}^M h_m(x).
|
||||
$$
|
||||
|
||||
<p>In the steepest decent approach we approximate \( h_m(x) = -\rho_m g_m(x) \), where \( \rho_m \) is a scalar and \( g_m(x) \) the gradient defined as</p>
|
||||
$$
|
||||
g_m(x_i) = \left[ \frac{\partial {\cal L}(y_i, f(x_i))}{\partial f(x_i)}\right]_{f(x_i)=f_{m-1}(x_i)}.
|
||||
$$
|
||||
|
||||
<p>With the new gradient we can update \( f_m(x) = f_{m-1}(x) -\rho_m g_m(x) \). Using the above squared-error function we see that
|
||||
the gradient is \( g_m(x_i) = -2(y_i-f(x_i)) \).
|
||||
<p>In order to understand the method, let us illustrate its basics by
|
||||
bringing back the essential steps in linear regression, where our cost
|
||||
function was the least squares function.
|
||||
</p>
|
||||
|
||||
<p>Choosing \( f_0(x)=0 \) we obtain \( g_m(x) = -2y_i \) and inserting this into the minimization problem for the cost function we have</p>
|
||||
$$
|
||||
(\rho_1) = \mathrm{argmin}_{\rho}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i+2\rho y_i)^2.
|
||||
$$
|
||||
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
@@ -216,6 +203,7 @@ $$
|
||||
<li><a href="._week45-bs018.html">19</a></li>
|
||||
<li><a href="._week45-bs019.html">20</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs014.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,19 +169,35 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0014"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="steepest-descent-example" class="anchor">Steepest Descent Example </h2>
|
||||
<h2 id="the-squared-error-again-steepest-descent" class="anchor">The Squared-Error again! Steepest Descent </h2>
|
||||
|
||||
<p>We start again with our cost function \( {\cal C}(\boldsymbol{y}m\boldsymbol{f})=\sum_{i=0}^{n-1}{\cal L}(y_i, f(x_i)) \) where we want to minimize
|
||||
This means that for every iteration, we need to optimize
|
||||
</p>
|
||||
|
||||
<p>Optimizing with respect to \( \rho \) we obtain (taking the derivative) that \( \rho_1 = -1/2 \). We have then that</p>
|
||||
$$
|
||||
f_1(x) = f_{0}(x) -\rho_1 g_1(x)=-y_i.
|
||||
(\hat{\boldsymbol{f}}) = \mathrm{argmin}_{\boldsymbol{f}}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f(x_i))^2.
|
||||
$$
|
||||
|
||||
<p>We can then proceed and compute</p>
|
||||
<p>We define a real function \( h_m(x) \) that defines our final function \( f_M(x) \) as</p>
|
||||
$$
|
||||
g_2(x_i) = \left[ \frac{\partial {\cal L}(y_i, f(x_i))}{\partial f(x_i)}\right]_{f(x_i)=f_{1}(x_i)=y_i}=-4y_i,
|
||||
f_M(x) = \sum_{m=0}^M h_m(x).
|
||||
$$
|
||||
|
||||
<p>In the steepest decent approach we approximate \( h_m(x) = -\rho_m g_m(x) \), where \( \rho_m \) is a scalar and \( g_m(x) \) the gradient defined as</p>
|
||||
$$
|
||||
g_m(x_i) = \left[ \frac{\partial {\cal L}(y_i, f(x_i))}{\partial f(x_i)}\right]_{f(x_i)=f_{m-1}(x_i)}.
|
||||
$$
|
||||
|
||||
<p>With the new gradient we can update \( f_m(x) = f_{m-1}(x) -\rho_m g_m(x) \). Using the above squared-error function we see that
|
||||
the gradient is \( g_m(x_i) = -2(y_i-f(x_i)) \).
|
||||
</p>
|
||||
|
||||
<p>Choosing \( f_0(x)=0 \) we obtain \( g_m(x) = -2y_i \) and inserting this into the minimization problem for the cost function we have</p>
|
||||
$$
|
||||
(\rho_1) = \mathrm{argmin}_{\rho}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i+2\rho y_i)^2.
|
||||
$$
|
||||
|
||||
<p>and find a new value for \( \rho_2=-1/2 \) and continue till we have reached \( m=M \). We can modify the steepest descent method, or steepest boosting, by introducing what is called <b>gradient boosting</b>. </p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -199,6 +220,7 @@ $$
|
||||
<li><a href="._week45-bs018.html">19</a></li>
|
||||
<li><a href="._week45-bs019.html">20</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs015.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,29 +169,20 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0015"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="gradient-boosting-algorithm" class="anchor">Gradient Boosting, algorithm </h2>
|
||||
<h2 id="steepest-descent-example" class="anchor">Steepest Descent Example </h2>
|
||||
|
||||
<p>Steepest descent is however not much used, since it only optimizes \( f \) at a fixed set of \( n \) points,
|
||||
so we do not learn a function that can generalize. However, we can modify the algorithm by
|
||||
fitting a weak learner to approximate the negative gradient signal.
|
||||
</p>
|
||||
|
||||
<p>Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard squared-error function</p>
|
||||
<p>Optimizing with respect to \( \rho \) we obtain (taking the derivative) that \( \rho_1 = -1/2 \). We have then that</p>
|
||||
$$
|
||||
C(\boldsymbol{y},\boldsymbol{f})=\sum_{i=0}^{n-1}(y_i-f(x_i))^2.
|
||||
f_1(x) = f_{0}(x) -\rho_1 g_1(x)=-y_i.
|
||||
$$
|
||||
|
||||
<p>The way we proceed in an iterative fashion is to</p>
|
||||
<ol>
|
||||
<li> Initialize our estimate \( f_0(x) \).</li>
|
||||
<li> For \( m=1:M \), we
|
||||
<ol type="a"></li>
|
||||
<li> compute the negative gradient vector \( \boldsymbol{u}_m = -\partial C(\boldsymbol{y},\boldsymbol{f})/\partial \boldsymbol{f}(x) \) at \( f(x) = f_{m-1}(x) \);</li>
|
||||
<li> fit the so-called base-learner to the negative gradient \( h_m(u_m,x) \);</li>
|
||||
<li> update the estimate \( f_m(x) = f_{m-1}(x)+h_m(u_m,x) \);</li>
|
||||
</ol>
|
||||
<li> The final estimate is then \( f_M(x) = \sum_{m=1}^M h_m(u_m,x) \).</li>
|
||||
</ol>
|
||||
<p>We can then proceed and compute</p>
|
||||
$$
|
||||
g_2(x_i) = \left[ \frac{\partial {\cal L}(y_i, f(x_i))}{\partial f(x_i)}\right]_{f(x_i)=f_{1}(x_i)=y_i}=-4y_i,
|
||||
$$
|
||||
|
||||
<p>and find a new value for \( \rho_2=-1/2 \) and continue till we have reached \( m=M \). We can modify the steepest descent method, or steepest boosting, by introducing what is called <b>gradient boosting</b>. </p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
@@ -207,6 +203,7 @@ $$
|
||||
<li><a href="._week45-bs018.html">19</a></li>
|
||||
<li><a href="._week45-bs019.html">20</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs016.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,71 +169,29 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0016"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="gradient-boosting-examples-of-regression" class="anchor">Gradient Boosting, Examples of Regression </h2>
|
||||
<h2 id="gradient-boosting-algorithm" class="anchor">Gradient Boosting, algorithm </h2>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> GradientBoostingRegressor
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> mean_squared_error
|
||||
|
||||
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
maxdegree <span style="color: #666666">=</span> <span style="color: #666666">6</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Make data set.</span>
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">-3</span>, <span style="color: #666666">3</span>, n)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x<span style="color: #666666">**2</span>) <span style="color: #666666">+</span> <span style="color: #666666">1.5</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(x<span style="color: #666666">-2</span>)<span style="color: #666666">**2</span>)<span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal(<span style="color: #666666">0</span>, <span style="color: #666666">0.1</span>, x<span style="color: #666666">.</span>shape)
|
||||
|
||||
error <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
variance <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
polydegree <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(x, y, test_size<span style="color: #666666">=0.2</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> degree <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1</span>,maxdegree):
|
||||
model <span style="color: #666666">=</span> GradientBoostingRegressor(max_depth<span style="color: #666666">=</span>degree, n_estimators<span style="color: #666666">=100</span>, learning_rate<span style="color: #666666">=1.0</span>)
|
||||
model<span style="color: #666666">.</span>fit(X_train,y_train)
|
||||
y_pred <span style="color: #666666">=</span> model<span style="color: #666666">.</span>predict(X_test)
|
||||
polydegree[degree] <span style="color: #666666">=</span> degree
|
||||
error[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>mean((y_test <span style="color: #666666">-</span> y_pred)<span style="color: #666666">**2</span>) )
|
||||
bias[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( (y_test <span style="color: #666666">-</span> np<span style="color: #666666">.</span>mean(y_pred))<span style="color: #666666">**2</span> )
|
||||
variance[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>var(y_pred) )
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Max depth:'</span>, degree)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Error:'</span>, error[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Bias^2:'</span>, bias[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Var:'</span>, variance[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'</span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> >= </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> + </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> = </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121">'</span><span style="color: #666666">.</span>format(error[degree], bias[degree], variance[degree], bias[degree]<span style="color: #666666">+</span>variance[degree]))
|
||||
|
||||
plt<span style="color: #666666">.</span>xlim(<span style="color: #666666">1</span>,maxdegree<span style="color: #666666">-1</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, error, label<span style="color: #666666">=</span><span style="color: #BA2121">'Error'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, bias, label<span style="color: #666666">=</span><span style="color: #BA2121">'bias'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, variance, label<span style="color: #666666">=</span><span style="color: #BA2121">'Variance'</span>)
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
save_fig(<span style="color: #BA2121">"gdregression"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<p>Steepest descent is however not much used, since it only optimizes \( f \) at a fixed set of \( n \) points,
|
||||
so we do not learn a function that can generalize. However, we can modify the algorithm by
|
||||
fitting a weak learner to approximate the negative gradient signal.
|
||||
</p>
|
||||
|
||||
<p>Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard squared-error function</p>
|
||||
$$
|
||||
C(\boldsymbol{y},\boldsymbol{f})=\sum_{i=0}^{n-1}(y_i-f(x_i))^2.
|
||||
$$
|
||||
|
||||
<p>The way we proceed in an iterative fashion is to</p>
|
||||
<ol>
|
||||
<li> Initialize our estimate \( f_0(x) \).</li>
|
||||
<li> For \( m=1:M \), we
|
||||
<ol type="a"></li>
|
||||
<li> compute the negative gradient vector \( \boldsymbol{u}_m = -\partial C(\boldsymbol{y},\boldsymbol{f})/\partial \boldsymbol{f}(x) \) at \( f(x) = f_{m-1}(x) \);</li>
|
||||
<li> fit the so-called base-learner to the negative gradient \( h_m(u_m,x) \);</li>
|
||||
<li> update the estimate \( f_m(x) = f_{m-1}(x)+h_m(u_m,x) \);</li>
|
||||
</ol>
|
||||
<li> The final estimate is then \( f_M(x) = \sum_{m=1}^M h_m(u_m,x) \).</li>
|
||||
</ol>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
@@ -248,6 +211,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._week45-bs018.html">19</a></li>
|
||||
<li><a href="._week45-bs019.html">20</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs017.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,7 +169,7 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0017"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="gradient-boosting-classification-example" class="anchor">Gradient Boosting, Classification Example </h2>
|
||||
<h2 id="gradient-boosting-examples-of-regression" class="anchor">Gradient Boosting, Examples of Regression </h2>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
@@ -174,43 +179,44 @@ MathJax.Hub.Config({
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> GradientBoostingRegressor
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> GradientBoostingClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> mean_squared_error
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Load the data</span>
|
||||
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
||||
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
maxdegree <span style="color: #666666">=</span> <span style="color: #666666">6</span>
|
||||
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
||||
<span style="color: #008000">print</span>(X_train<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #008000">print</span>(X_test<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic">#now scale the data</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
scaler<span style="color: #666666">.</span>fit(X_train)
|
||||
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
||||
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
||||
<span style="color: #408080; font-style: italic"># Make data set.</span>
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">-3</span>, <span style="color: #666666">3</span>, n)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x<span style="color: #666666">**2</span>) <span style="color: #666666">+</span> <span style="color: #666666">1.5</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(x<span style="color: #666666">-2</span>)<span style="color: #666666">**2</span>)<span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal(<span style="color: #666666">0</span>, <span style="color: #666666">0.1</span>, x<span style="color: #666666">.</span>shape)
|
||||
|
||||
gd_clf <span style="color: #666666">=</span> GradientBoostingClassifier(max_depth<span style="color: #666666">=3</span>, n_estimators<span style="color: #666666">=100</span>, learning_rate<span style="color: #666666">=1.0</span>)
|
||||
gd_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #408080; font-style: italic">#Cross validation</span>
|
||||
accuracy <span style="color: #666666">=</span> cross_validate(gd_clf,X_test_scaled,y_test,cv<span style="color: #666666">=10</span>)[<span style="color: #BA2121">'test_score'</span>]
|
||||
<span style="color: #008000">print</span>(accuracy)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Random Forests and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(gd_clf<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
error <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
variance <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
polydegree <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(x, y, test_size<span style="color: #666666">=0.2</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
y_pred <span style="color: #666666">=</span> gd_clf<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
save_fig(<span style="color: #BA2121">"gdclassiffierconfusion"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
y_probas <span style="color: #666666">=</span> gd_clf<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
||||
save_fig(<span style="color: #BA2121">"gdclassiffierroc"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
||||
save_fig(<span style="color: #BA2121">"gdclassiffiercgain"</span>)
|
||||
<span style="color: #008000; font-weight: bold">for</span> degree <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1</span>,maxdegree):
|
||||
model <span style="color: #666666">=</span> GradientBoostingRegressor(max_depth<span style="color: #666666">=</span>degree, n_estimators<span style="color: #666666">=100</span>, learning_rate<span style="color: #666666">=1.0</span>)
|
||||
model<span style="color: #666666">.</span>fit(X_train,y_train)
|
||||
y_pred <span style="color: #666666">=</span> model<span style="color: #666666">.</span>predict(X_test)
|
||||
polydegree[degree] <span style="color: #666666">=</span> degree
|
||||
error[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>mean((y_test <span style="color: #666666">-</span> y_pred)<span style="color: #666666">**2</span>) )
|
||||
bias[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( (y_test <span style="color: #666666">-</span> np<span style="color: #666666">.</span>mean(y_pred))<span style="color: #666666">**2</span> )
|
||||
variance[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>var(y_pred) )
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Max depth:'</span>, degree)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Error:'</span>, error[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Bias^2:'</span>, bias[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Var:'</span>, variance[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'</span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> >= </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> + </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> = </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121">'</span><span style="color: #666666">.</span>format(error[degree], bias[degree], variance[degree], bias[degree]<span style="color: #666666">+</span>variance[degree]))
|
||||
|
||||
plt<span style="color: #666666">.</span>xlim(<span style="color: #666666">1</span>,maxdegree<span style="color: #666666">-1</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, error, label<span style="color: #666666">=</span><span style="color: #BA2121">'Error'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, bias, label<span style="color: #666666">=</span><span style="color: #BA2121">'bias'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, variance, label<span style="color: #666666">=</span><span style="color: #BA2121">'Variance'</span>)
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
save_fig(<span style="color: #BA2121">"gdregression"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre>
|
||||
</div>
|
||||
@@ -246,6 +252,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._week45-bs018.html">19</a></li>
|
||||
<li><a href="._week45-bs019.html">20</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs018.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,22 +169,69 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0018"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="xgboost-extreme-gradient-boosting" class="anchor">XGBoost: Extreme Gradient Boosting </h2>
|
||||
<h2 id="gradient-boosting-classification-example" class="anchor">Gradient Boosting, Classification Example </h2>
|
||||
|
||||
<p><a href="https://github.com/dmlc/xgboost" target="_self">XGBoost</a> or Extreme Gradient
|
||||
Boosting, is an optimized distributed gradient boosting library
|
||||
designed to be highly efficient, flexible and portable. It implements
|
||||
machine learning algorithms under the Gradient Boosting
|
||||
framework. XGBoost provides a parallel tree boosting that solve many
|
||||
data science problems in a fast and accurate way. See the <a href="https://arxiv.org/abs/1603.02754" target="_self">article by Chen and Guestrin</a>.
|
||||
</p>
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> GradientBoostingClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
|
||||
|
||||
<p>The authors design and build a highly scalable end-to-end tree
|
||||
boosting system. It has a theoretically justified weighted quantile
|
||||
sketch for efficient proposal calculation. It introduces a novel sparsity-aware algorithm for parallel tree learning and an effective cache-aware block structure for out-of-core tree learning.
|
||||
</p>
|
||||
<span style="color: #408080; font-style: italic"># Load the data</span>
|
||||
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
||||
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
||||
<span style="color: #008000">print</span>(X_train<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #008000">print</span>(X_test<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic">#now scale the data</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
scaler<span style="color: #666666">.</span>fit(X_train)
|
||||
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
||||
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
||||
|
||||
gd_clf <span style="color: #666666">=</span> GradientBoostingClassifier(max_depth<span style="color: #666666">=3</span>, n_estimators<span style="color: #666666">=100</span>, learning_rate<span style="color: #666666">=1.0</span>)
|
||||
gd_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #408080; font-style: italic">#Cross validation</span>
|
||||
accuracy <span style="color: #666666">=</span> cross_validate(gd_clf,X_test_scaled,y_test,cv<span style="color: #666666">=10</span>)[<span style="color: #BA2121">'test_score'</span>]
|
||||
<span style="color: #008000">print</span>(accuracy)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Random Forests and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(gd_clf<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
y_pred <span style="color: #666666">=</span> gd_clf<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
save_fig(<span style="color: #BA2121">"gdclassiffierconfusion"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
y_probas <span style="color: #666666">=</span> gd_clf<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
||||
save_fig(<span style="color: #BA2121">"gdclassiffierroc"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
||||
save_fig(<span style="color: #BA2121">"gdclassiffiercgain"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>It is now the algorithm which wins essentially all ML competitions!!!</p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -198,6 +250,7 @@ sketch for efficient proposal calculation. It introduces a novel sparsity-aware
|
||||
<li class="active"><a href="._week45-bs018.html">19</a></li>
|
||||
<li><a href="._week45-bs019.html">20</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs019.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,71 +169,22 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0019"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="regression-case" class="anchor">Regression Case </h2>
|
||||
<h2 id="xgboost-extreme-gradient-boosting" class="anchor">XGBoost: Extreme Gradient Boosting </h2>
|
||||
|
||||
<p><a href="https://github.com/dmlc/xgboost" target="_self">XGBoost</a> or Extreme Gradient
|
||||
Boosting, is an optimized distributed gradient boosting library
|
||||
designed to be highly efficient, flexible and portable. It implements
|
||||
machine learning algorithms under the Gradient Boosting
|
||||
framework. XGBoost provides a parallel tree boosting that solve many
|
||||
data science problems in a fast and accurate way. See the <a href="https://arxiv.org/abs/1603.02754" target="_self">article by Chen and Guestrin</a>.
|
||||
</p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">xgboost</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">xgb</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> mean_squared_error
|
||||
|
||||
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
maxdegree <span style="color: #666666">=</span> <span style="color: #666666">6</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Make data set.</span>
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">-3</span>, <span style="color: #666666">3</span>, n)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x<span style="color: #666666">**2</span>) <span style="color: #666666">+</span> <span style="color: #666666">1.5</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(x<span style="color: #666666">-2</span>)<span style="color: #666666">**2</span>)<span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal(<span style="color: #666666">0</span>, <span style="color: #666666">0.1</span>, x<span style="color: #666666">.</span>shape)
|
||||
|
||||
error <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
variance <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
polydegree <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(x, y, test_size<span style="color: #666666">=0.2</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> degree <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(maxdegree):
|
||||
model <span style="color: #666666">=</span> xgb<span style="color: #666666">.</span>XGBRegressor(objective <span style="color: #666666">=</span><span style="color: #BA2121">'reg:squarederror'</span>, colsaobjective <span style="color: #666666">=</span><span style="color: #BA2121">'reg:squarederror'</span>, colsample_bytree <span style="color: #666666">=</span> <span style="color: #666666">0.3</span>, learning_rate <span style="color: #666666">=</span> <span style="color: #666666">0.1</span>,max_depth <span style="color: #666666">=</span> degree, alpha <span style="color: #666666">=</span> <span style="color: #666666">10</span>, n_estimators <span style="color: #666666">=</span> <span style="color: #666666">200</span>)
|
||||
|
||||
model<span style="color: #666666">.</span>fit(X_train,y_train)
|
||||
y_pred <span style="color: #666666">=</span> model<span style="color: #666666">.</span>predict(X_test)
|
||||
polydegree[degree] <span style="color: #666666">=</span> degree
|
||||
error[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>mean((y_test <span style="color: #666666">-</span> y_pred)<span style="color: #666666">**2</span>) )
|
||||
bias[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( (y_test <span style="color: #666666">-</span> np<span style="color: #666666">.</span>mean(y_pred))<span style="color: #666666">**2</span> )
|
||||
variance[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>var(y_pred) )
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Max depth:'</span>, degree)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Error:'</span>, error[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Bias^2:'</span>, bias[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Var:'</span>, variance[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'</span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> >= </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> + </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> = </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121">'</span><span style="color: #666666">.</span>format(error[degree], bias[degree], variance[degree], bias[degree]<span style="color: #666666">+</span>variance[degree]))
|
||||
|
||||
plt<span style="color: #666666">.</span>xlim(<span style="color: #666666">1</span>,maxdegree<span style="color: #666666">-1</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, error, label<span style="color: #666666">=</span><span style="color: #BA2121">'Error'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, bias, label<span style="color: #666666">=</span><span style="color: #BA2121">'bias'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, variance, label<span style="color: #666666">=</span><span style="color: #BA2121">'Variance'</span>)
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<p>The authors design and build a highly scalable end-to-end tree
|
||||
boosting system. It has a theoretically justified weighted quantile
|
||||
sketch for efficient proposal calculation. It introduces a novel sparsity-aware algorithm for parallel tree learning and an effective cache-aware block structure for out-of-core tree learning.
|
||||
</p>
|
||||
|
||||
<p>It is now the algorithm which wins essentially all ML competitions!!!</p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -246,6 +202,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._week45-bs018.html">19</a></li>
|
||||
<li class="active"><a href="._week45-bs019.html">20</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs020.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,9 +169,8 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0020"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="xgboost-on-the-cancer-data" class="anchor">Xgboost on the Cancer Data </h2>
|
||||
<h2 id="regression-case" class="anchor">Regression Case </h2>
|
||||
|
||||
<p>As you will see from the confusion matrix below, XGBoots does an excellent job on the Wisconsin cancer data and outperforms essentially all agorithms we have discussed till now. </p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
@@ -176,54 +180,44 @@ MathJax.Hub.Config({
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> LabelEncoder
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">xgboost</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">xgb</span>
|
||||
<span style="color: #408080; font-style: italic"># Load the data</span>
|
||||
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
||||
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
||||
<span style="color: #008000">print</span>(X_train<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #008000">print</span>(X_test<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic">#now scale the data</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
scaler<span style="color: #666666">.</span>fit(X_train)
|
||||
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
||||
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
||||
|
||||
xg_clf <span style="color: #666666">=</span> xgb<span style="color: #666666">.</span>XGBClassifier()
|
||||
xg_clf<span style="color: #666666">.</span>fit(X_train_scaled,y_train)
|
||||
|
||||
y_test <span style="color: #666666">=</span> xg_clf<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Random Forests and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(xg_clf<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
y_pred <span style="color: #666666">=</span> xg_clf<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
save_fig(<span style="color: #BA2121">"xdclassiffierconfusion"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
y_probas <span style="color: #666666">=</span> xg_clf<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
||||
save_fig(<span style="color: #BA2121">"xdclassiffierroc"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
||||
save_fig(<span style="color: #BA2121">"gdclassiffiercgain"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> mean_squared_error
|
||||
|
||||
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
maxdegree <span style="color: #666666">=</span> <span style="color: #666666">6</span>
|
||||
|
||||
xgb<span style="color: #666666">.</span>plot_tree(xg_clf,num_trees<span style="color: #666666">=0</span>)
|
||||
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">'figure.figsize'</span>] <span style="color: #666666">=</span> [<span style="color: #666666">50</span>, <span style="color: #666666">10</span>]
|
||||
save_fig(<span style="color: #BA2121">"xgtree"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
<span style="color: #408080; font-style: italic"># Make data set.</span>
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">-3</span>, <span style="color: #666666">3</span>, n)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x<span style="color: #666666">**2</span>) <span style="color: #666666">+</span> <span style="color: #666666">1.5</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(x<span style="color: #666666">-2</span>)<span style="color: #666666">**2</span>)<span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal(<span style="color: #666666">0</span>, <span style="color: #666666">0.1</span>, x<span style="color: #666666">.</span>shape)
|
||||
|
||||
xgb<span style="color: #666666">.</span>plot_importance(xg_clf)
|
||||
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">'figure.figsize'</span>] <span style="color: #666666">=</span> [<span style="color: #666666">5</span>, <span style="color: #666666">5</span>]
|
||||
save_fig(<span style="color: #BA2121">"xgparams"</span>)
|
||||
error <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
variance <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
polydegree <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(x, y, test_size<span style="color: #666666">=0.2</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> degree <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(maxdegree):
|
||||
model <span style="color: #666666">=</span> xgb<span style="color: #666666">.</span>XGBRegressor(objective <span style="color: #666666">=</span><span style="color: #BA2121">'reg:squarederror'</span>, colsaobjective <span style="color: #666666">=</span><span style="color: #BA2121">'reg:squarederror'</span>, colsample_bytree <span style="color: #666666">=</span> <span style="color: #666666">0.3</span>, learning_rate <span style="color: #666666">=</span> <span style="color: #666666">0.1</span>,max_depth <span style="color: #666666">=</span> degree, alpha <span style="color: #666666">=</span> <span style="color: #666666">10</span>, n_estimators <span style="color: #666666">=</span> <span style="color: #666666">200</span>)
|
||||
|
||||
model<span style="color: #666666">.</span>fit(X_train,y_train)
|
||||
y_pred <span style="color: #666666">=</span> model<span style="color: #666666">.</span>predict(X_test)
|
||||
polydegree[degree] <span style="color: #666666">=</span> degree
|
||||
error[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>mean((y_test <span style="color: #666666">-</span> y_pred)<span style="color: #666666">**2</span>) )
|
||||
bias[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( (y_test <span style="color: #666666">-</span> np<span style="color: #666666">.</span>mean(y_pred))<span style="color: #666666">**2</span> )
|
||||
variance[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>var(y_pred) )
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Max depth:'</span>, degree)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Error:'</span>, error[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Bias^2:'</span>, bias[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Var:'</span>, variance[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'</span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> >= </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> + </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> = </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121">'</span><span style="color: #666666">.</span>format(error[degree], bias[degree], variance[degree], bias[degree]<span style="color: #666666">+</span>variance[degree]))
|
||||
|
||||
plt<span style="color: #666666">.</span>xlim(<span style="color: #666666">1</span>,maxdegree<span style="color: #666666">-1</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, error, label<span style="color: #666666">=</span><span style="color: #BA2121">'Error'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, bias, label<span style="color: #666666">=</span><span style="color: #BA2121">'bias'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, variance, label<span style="color: #666666">=</span><span style="color: #BA2121">'Variance'</span>)
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre>
|
||||
</div>
|
||||
@@ -256,6 +250,8 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._week45-bs018.html">19</a></li>
|
||||
<li><a href="._week45-bs019.html">20</a></li>
|
||||
<li class="active"><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs021.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
</div> <!-- end container -->
|
||||
|
||||
@@ -37,175 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Decision trees, overarching aims',
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'decision-trees-overarching-aims'),
|
||||
('Basics of a tree', 2, None, 'basics-of-a-tree'),
|
||||
('A Sketch of a Tree, Regression problem',
|
||||
2,
|
||||
None,
|
||||
'a-sketch-of-a-tree-regression-problem'),
|
||||
('A Sketch of a Tree, Classification problem',
|
||||
2,
|
||||
None,
|
||||
'a-sketch-of-a-tree-classification-problem'),
|
||||
('A typical Decision Tree with its pertinent Jargon, '
|
||||
'Classification Problem',
|
||||
2,
|
||||
None,
|
||||
'a-typical-decision-tree-with-its-pertinent-jargon-classification-problem'),
|
||||
('General Features', 2, None, 'general-features'),
|
||||
('How do we set it up?', 2, None, 'how-do-we-set-it-up'),
|
||||
('Decision trees and Regression',
|
||||
2,
|
||||
None,
|
||||
'decision-trees-and-regression'),
|
||||
('Building a tree, regression',
|
||||
2,
|
||||
None,
|
||||
'building-a-tree-regression'),
|
||||
('A top-down approach, recursive binary splitting',
|
||||
2,
|
||||
None,
|
||||
'a-top-down-approach-recursive-binary-splitting'),
|
||||
('Making a tree', 2, None, 'making-a-tree'),
|
||||
('Pruning the tree', 2, None, 'pruning-the-tree'),
|
||||
('Cost complexity pruning', 2, None, 'cost-complexity-pruning'),
|
||||
('Schematic Regression Procedure',
|
||||
2,
|
||||
None,
|
||||
'schematic-regression-procedure'),
|
||||
('A Classification Tree', 2, None, 'a-classification-tree'),
|
||||
('Growing a classification tree',
|
||||
2,
|
||||
None,
|
||||
'growing-a-classification-tree'),
|
||||
('Classification tree, how to split nodes',
|
||||
2,
|
||||
None,
|
||||
'classification-tree-how-to-split-nodes'),
|
||||
('Gini Index (or Coefficient or Impurity)',
|
||||
2,
|
||||
None,
|
||||
'gini-index-or-coefficient-or-impurity'),
|
||||
('Why binary splits?', 2, None, 'why-binary-splits'),
|
||||
('Computing a Tree using the Gini Index',
|
||||
2,
|
||||
None,
|
||||
'computing-a-tree-using-the-gini-index'),
|
||||
('The Table', 2, None, 'the-table'),
|
||||
('Computing the various Gini Indices',
|
||||
2,
|
||||
None,
|
||||
'computing-the-various-gini-indices'),
|
||||
('Computing the various Gini Indices, Hours slept',
|
||||
2,
|
||||
None,
|
||||
'computing-the-various-gini-indices-hours-slept'),
|
||||
('Computing the various Gini Indices, Hours studied',
|
||||
2,
|
||||
None,
|
||||
'computing-the-various-gini-indices-hours-studied'),
|
||||
('A possible code using Scikit-Learn',
|
||||
2,
|
||||
None,
|
||||
'a-possible-code-using-scikit-learn'),
|
||||
('Visualizing Trees, More examples',
|
||||
2,
|
||||
None,
|
||||
'visualizing-trees-more-examples'),
|
||||
('Visualizing the Tree, The Moons',
|
||||
2,
|
||||
None,
|
||||
'visualizing-the-tree-the-moons'),
|
||||
('Other ways of visualizing the trees',
|
||||
2,
|
||||
None,
|
||||
'other-ways-of-visualizing-the-trees'),
|
||||
('Printing out as text', 2, None, 'printing-out-as-text'),
|
||||
('Algorithms for Setting up Decision Trees',
|
||||
2,
|
||||
None,
|
||||
'algorithms-for-setting-up-decision-trees'),
|
||||
('The CART algorithm for Classification',
|
||||
2,
|
||||
None,
|
||||
'the-cart-algorithm-for-classification'),
|
||||
('The CART algorithm for Regression',
|
||||
2,
|
||||
None,
|
||||
'the-cart-algorithm-for-regression'),
|
||||
('Computing the Gini index', 2, None, 'computing-the-gini-index'),
|
||||
('Simple Python Code to read in Data and perform Classification',
|
||||
2,
|
||||
None,
|
||||
'simple-python-code-to-read-in-data-and-perform-classification'),
|
||||
('Computing the Gini Factor',
|
||||
2,
|
||||
None,
|
||||
'computing-the-gini-factor'),
|
||||
('Entropy and the ID3 algorithm',
|
||||
2,
|
||||
None,
|
||||
'entropy-and-the-id3-algorithm'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'cancer-data-again-now-with-decision-trees-and-other-methods'),
|
||||
('Another example, the moons again',
|
||||
2,
|
||||
None,
|
||||
'another-example-the-moons-again'),
|
||||
('Playing around with regions',
|
||||
2,
|
||||
None,
|
||||
'playing-around-with-regions'),
|
||||
('Regression trees', 2, None, 'regression-trees'),
|
||||
('Final regressor code', 2, None, 'final-regressor-code'),
|
||||
('Pros and cons of trees, pros',
|
||||
2,
|
||||
None,
|
||||
'pros-and-cons-of-trees-pros'),
|
||||
('Disadvantages', 2, None, 'disadvantages'),
|
||||
('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
|
||||
'Boosting, Meet the Jungle of Methods',
|
||||
2,
|
||||
None,
|
||||
'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'),
|
||||
('An Overview of Ensemble Methods',
|
||||
2,
|
||||
None,
|
||||
'an-overview-of-ensemble-methods'),
|
||||
('Bagging', 2, None, 'bagging'),
|
||||
('More bagging', 2, None, 'more-bagging'),
|
||||
('Making your own Bootstrap: Changing the Level of the Decision '
|
||||
'Tree',
|
||||
2,
|
||||
None,
|
||||
'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'),
|
||||
('Why Voting?', 2, None, 'why-voting'),
|
||||
('Tossing coins', 2, None, 'tossing-coins'),
|
||||
('Standard imports first', 2, None, 'standard-imports-first'),
|
||||
('Simple Voting Example, head or tail',
|
||||
2,
|
||||
None,
|
||||
'simple-voting-example-head-or-tail'),
|
||||
('Using the Voting Classifier',
|
||||
2,
|
||||
None,
|
||||
'using-the-voting-classifier'),
|
||||
('Voting and Bagging', 2, None, 'voting-and-bagging'),
|
||||
('Random forests', 2, None, 'random-forests'),
|
||||
('Random Forest Algorithm', 2, None, 'random-forest-algorithm'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'random-forests-compared-with-other-methods-on-the-cancer-data'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'compare-bagging-on-trees-with-random-forests'),
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -303,83 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#decision-trees-overarching-aims" style="font-size: 80%;">Decision trees, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#basics-of-a-tree" style="font-size: 80%;">Basics of a tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#a-sketch-of-a-tree-regression-problem" style="font-size: 80%;">A Sketch of a Tree, Regression problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#a-sketch-of-a-tree-classification-problem" style="font-size: 80%;">A Sketch of a Tree, Classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#a-typical-decision-tree-with-its-pertinent-jargon-classification-problem" style="font-size: 80%;">A typical Decision Tree with its pertinent Jargon, Classification Problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#general-features" style="font-size: 80%;">General Features</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#how-do-we-set-it-up" style="font-size: 80%;">How do we set it up?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#decision-trees-and-regression" style="font-size: 80%;">Decision trees and Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#building-a-tree-regression" style="font-size: 80%;">Building a tree, regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#a-top-down-approach-recursive-binary-splitting" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#making-a-tree" style="font-size: 80%;">Making a tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#pruning-the-tree" style="font-size: 80%;">Pruning the tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#cost-complexity-pruning" style="font-size: 80%;">Cost complexity pruning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#schematic-regression-procedure" style="font-size: 80%;">Schematic Regression Procedure</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#a-classification-tree" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#growing-a-classification-tree" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#classification-tree-how-to-split-nodes" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#gini-index-or-coefficient-or-impurity" style="font-size: 80%;">Gini Index (or Coefficient or Impurity)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#why-binary-splits" style="font-size: 80%;">Why binary splits?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#computing-a-tree-using-the-gini-index" style="font-size: 80%;">Computing a Tree using the Gini Index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs022.html#the-table" style="font-size: 80%;">The Table</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs023.html#computing-the-various-gini-indices" style="font-size: 80%;">Computing the various Gini Indices</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs024.html#computing-the-various-gini-indices-hours-slept" style="font-size: 80%;">Computing the various Gini Indices, Hours slept</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs025.html#computing-the-various-gini-indices-hours-studied" style="font-size: 80%;">Computing the various Gini Indices, Hours studied</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs026.html#a-possible-code-using-scikit-learn" style="font-size: 80%;">A possible code using Scikit-Learn</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs027.html#visualizing-trees-more-examples" style="font-size: 80%;">Visualizing Trees, More examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs028.html#visualizing-the-tree-the-moons" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs029.html#other-ways-of-visualizing-the-trees" style="font-size: 80%;">Other ways of visualizing the trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs030.html#printing-out-as-text" style="font-size: 80%;">Printing out as text</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs031.html#algorithms-for-setting-up-decision-trees" style="font-size: 80%;">Algorithms for Setting up Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs032.html#the-cart-algorithm-for-classification" style="font-size: 80%;">The CART algorithm for Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs033.html#the-cart-algorithm-for-regression" style="font-size: 80%;">The CART algorithm for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs034.html#computing-the-gini-index" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs035.html#simple-python-code-to-read-in-data-and-perform-classification" style="font-size: 80%;">Simple Python Code to read in Data and perform Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs036.html#computing-the-gini-factor" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs037.html#entropy-and-the-id3-algorithm" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs038.html#cancer-data-again-now-with-decision-trees-and-other-methods" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs039.html#another-example-the-moons-again" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs040.html#playing-around-with-regions" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs041.html#regression-trees" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs042.html#final-regressor-code" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs043.html#pros-and-cons-of-trees-pros" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs044.html#disadvantages" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs045.html#ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods" style="font-size: 80%;">Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs046.html#an-overview-of-ensemble-methods" style="font-size: 80%;">An Overview of Ensemble Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs047.html#bagging" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs048.html#more-bagging" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs049.html#making-your-own-bootstrap-changing-the-level-of-the-decision-tree" style="font-size: 80%;">Making your own Bootstrap: Changing the Level of the Decision Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs050.html#why-voting" style="font-size: 80%;">Why Voting?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs051.html#tossing-coins" style="font-size: 80%;">Tossing coins</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs052.html#standard-imports-first" style="font-size: 80%;">Standard imports first</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs053.html#simple-voting-example-head-or-tail" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs054.html#using-the-voting-classifier" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs055.html#voting-and-bagging" style="font-size: 80%;">Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs056.html#random-forests" style="font-size: 80%;">Random forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs057.html#random-forest-algorithm" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs058.html#random-forests-compared-with-other-methods-on-the-cancer-data" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs059.html#compare-bagging-on-trees-with-random-forests" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs060.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs061.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs062.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs063.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs064.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs065.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs066.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs067.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs068.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs069.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs070.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs071.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs072.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs073.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs074.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs075.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs076.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs077.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs078.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -391,23 +169,83 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
<a name="part0021"></a>
|
||||
<!-- !split -->
|
||||
<h2 id="computing-a-tree-using-the-gini-index" class="anchor">Computing a Tree using the Gini Index </h2>
|
||||
<h2 id="xgboost-on-the-cancer-data" class="anchor">Xgboost on the Cancer Data </h2>
|
||||
|
||||
<p>As you will see from the confusion matrix below, XGBoots does an excellent job on the Wisconsin cancer data and outperforms essentially all agorithms we have discussed till now. </p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> LabelEncoder
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">xgboost</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">xgb</span>
|
||||
<span style="color: #408080; font-style: italic"># Load the data</span>
|
||||
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
||||
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
||||
<span style="color: #008000">print</span>(X_train<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #008000">print</span>(X_test<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic">#now scale the data</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
scaler<span style="color: #666666">.</span>fit(X_train)
|
||||
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
||||
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
||||
|
||||
xg_clf <span style="color: #666666">=</span> xgb<span style="color: #666666">.</span>XGBClassifier()
|
||||
xg_clf<span style="color: #666666">.</span>fit(X_train_scaled,y_train)
|
||||
|
||||
y_test <span style="color: #666666">=</span> xg_clf<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Random Forests and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(xg_clf<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
y_pred <span style="color: #666666">=</span> xg_clf<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
save_fig(<span style="color: #BA2121">"xdclassiffierconfusion"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
y_probas <span style="color: #666666">=</span> xg_clf<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
||||
save_fig(<span style="color: #BA2121">"xdclassiffierroc"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
||||
save_fig(<span style="color: #BA2121">"gdclassiffiercgain"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
|
||||
xgb<span style="color: #666666">.</span>plot_tree(xg_clf,num_trees<span style="color: #666666">=0</span>)
|
||||
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">'figure.figsize'</span>] <span style="color: #666666">=</span> [<span style="color: #666666">50</span>, <span style="color: #666666">10</span>]
|
||||
save_fig(<span style="color: #BA2121">"xgtree"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
xgb<span style="color: #666666">.</span>plot_importance(xg_clf)
|
||||
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">'figure.figsize'</span>] <span style="color: #666666">=</span> [<span style="color: #666666">5</span>, <span style="color: #666666">5</span>]
|
||||
save_fig(<span style="color: #BA2121">"xgparams"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>Consider the following example with attributes/features and two
|
||||
possible outcomes (classes) for each attribute. Assume we wish to find some
|
||||
correlations between the average grade of a student as function of the
|
||||
number of hours studied and hours slept. We want also to correlate the
|
||||
grade in a given course with the general trend, whether the students
|
||||
recently has gotten grades below average or above.
|
||||
</p>
|
||||
|
||||
<p>We have three features/attributes</p>
|
||||
<ol>
|
||||
<li> Trend of average grades before present course, classified as either below or above the average grade of the whole class</li>
|
||||
<li> The number of hours studies, classified again as either higher (more than 3 hours per day) or lower . Here we have used a standard for one \( ECTS \) which is scaled to 25-30 hours of work for a semester which lasts 18 weeks, with 15 weeks of lectures and 3 weeks for exams, assuming a total of 30 ECTS per semester.</li>
|
||||
<li> The number of hours slept as high for more than \( 8 \) hours and below for less than 8 hours of sleep, classified again as either high or low</li>
|
||||
<li> The final grade whether it is above or below average</li>
|
||||
</ol>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
@@ -423,18 +261,6 @@ recently has gotten grades below average or above.
|
||||
<li><a href="._week45-bs019.html">20</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li class="active"><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs022.html">23</a></li>
|
||||
<li><a href="._week45-bs023.html">24</a></li>
|
||||
<li><a href="._week45-bs024.html">25</a></li>
|
||||
<li><a href="._week45-bs025.html">26</a></li>
|
||||
<li><a href="._week45-bs026.html">27</a></li>
|
||||
<li><a href="._week45-bs027.html">28</a></li>
|
||||
<li><a href="._week45-bs028.html">29</a></li>
|
||||
<li><a href="._week45-bs029.html">30</a></li>
|
||||
<li><a href="._week45-bs030.html">31</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs078.html">79</a></li>
|
||||
<li><a href="._week45-bs022.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
</div> <!-- end container -->
|
||||
|
||||
@@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -134,25 +138,26 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs001.html#overview-of-week-45" style="font-size: 80%;">Overview of week 45</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs002.html#brief-code-reminder-from-last-wekk" style="font-size: 80%;">Brief code reminder from last wekk</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs013.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs014.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs015.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs016.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs017.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs018.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs019.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs020.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week45-bs021.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -207,7 +212,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._week45-bs008.html">9</a></li>
|
||||
<li><a href="._week45-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week45-bs020.html">21</a></li>
|
||||
<li><a href="._week45-bs021.html">22</a></li>
|
||||
<li><a href="._week45-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -221,6 +221,122 @@ MathJax.Hub.Config({
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="brief-code-reminder-from-last-wekk">Brief code reminder from last wekk </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># Common imports</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">IPython.display</span> <span style="color: #8B008B; font-weight: bold">import</span> Image
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">pydot</span> <span style="color: #8B008B; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> DecisionTreeRegressor
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> export_graphviz
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> StandardScaler, OneHotEncoder
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.compose</span> <span style="color: #8B008B; font-weight: bold">import</span> ColumnTransformer
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">IPython.display</span> <span style="color: #8B008B; font-weight: bold">import</span> Image
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">pydot</span> <span style="color: #8B008B; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> BaggingClassifier
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span>
|
||||
|
||||
<span style="color: #228B22"># Where to save the figures and data files</span>
|
||||
PROJECT_ROOT_DIR = <span style="color: #CD5555">"Results"</span>
|
||||
FIGURE_ID = <span style="color: #CD5555">"Results/FigureFiles"</span>
|
||||
DATA_ID = <span style="color: #CD5555">"DataFiles/"</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #8B008B">not</span> os.path.exists(PROJECT_ROOT_DIR):
|
||||
os.mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #8B008B">not</span> os.path.exists(FIGURE_ID):
|
||||
os.makedirs(FIGURE_ID)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #8B008B">not</span> os.path.exists(DATA_ID):
|
||||
os.makedirs(DATA_ID)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">image_path</span>(fig_id):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> os.path.join(FIGURE_ID, fig_id)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">data_path</span>(dat_id):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> os.path.join(DATA_ID, dat_id)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">save_fig</span>(fig_id):
|
||||
plt.savefig(image_path(fig_id) + <span style="color: #CD5555">".png"</span>, <span style="color: #658b00">format</span>=<span style="color: #CD5555">'png'</span>)
|
||||
|
||||
<span style="color: #228B22"># Load the cancer data</span>
|
||||
cancer = load_breast_cancer()
|
||||
|
||||
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=<span style="color: #B452CD">0</span>)
|
||||
<span style="color: #658b00">print</span>(X_train.shape)
|
||||
<span style="color: #658b00">print</span>(X_test.shape)
|
||||
<span style="color: #228B22">#Scale the data</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> StandardScaler
|
||||
scaler = StandardScaler()
|
||||
scaler.fit(X_train)
|
||||
X_train_scaled = scaler.transform(X_train)
|
||||
X_test_scaled = scaler.transform(X_test)
|
||||
<span style="color: #228B22">#define methods</span>
|
||||
<span style="color: #228B22"># Logistic Regression</span>
|
||||
logreg.fit(X_train_scaled, y_train)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test set accuracy Logistic Regression with scaled data: {:.2f}"</span>.format(logreg.score(X_test_scaled,y_test)))
|
||||
<span style="color: #228B22"># Support Vector Machine</span>
|
||||
svm.fit(X_train_scaled, y_train)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test set accuracy SVM with scaled data: {:.2f}"</span>.format(logreg.score(X_test_scaled,y_test)))
|
||||
<span style="color: #228B22"># Decision Trees</span>
|
||||
deep_tree_clf.fit(X_train_scaled, y_train)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test set accuracy with Decision Trees and scaled data: {:.2f}"</span>.format(deep_tree_clf.score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> LabelEncoder
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> cross_validate
|
||||
<span style="color: #228B22"># Data set not specificied</span>
|
||||
<span style="color: #228B22">#Instantiate the model with 500 trees and entropy as splitting criteria</span>
|
||||
Random_Forest_model = RandomForestClassifier(n_estimators=<span style="color: #B452CD">500</span>,criterion=<span style="color: #CD5555">"entropy"</span>)
|
||||
Random_Forest_model.fit(X_train_scaled, y_train)
|
||||
<span style="color: #228B22">#Cross validation</span>
|
||||
accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=<span style="color: #B452CD">10</span>)[<span style="color: #CD5555">'test_score'</span>]
|
||||
<span style="color: #658b00">print</span>(accuracy)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test set accuracy with Random Forests and scaled data: {:.2f}"</span>.format(Random_Forest_model.score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">scikitplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">skplt</span>
|
||||
y_pred = Random_Forest_model.predict(X_test_scaled)
|
||||
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=<span style="color: #8B008B; font-weight: bold">True</span>)
|
||||
plt.show()
|
||||
y_probas = Random_Forest_model.predict_proba(X_test_scaled)
|
||||
skplt.metrics.plot_roc(y_test, y_probas)
|
||||
plt.show()
|
||||
skplt.metrics.plot_cumulative_gain(y_test, y_probas)
|
||||
plt.show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="boosting-a-bird-s-eye-view">Boosting, a Bird's Eye View </h2>
|
||||
|
||||
|
||||
@@ -64,6 +64,10 @@ div.toc p,a {
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -193,6 +197,122 @@ MathJax.Hub.Config({
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="brief-code-reminder-from-last-wekk">Brief code reminder from last wekk </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="line-height: 125%;"><span style="color: #228B22"># Common imports</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">IPython.display</span> <span style="color: #8B008B; font-weight: bold">import</span> Image
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">pydot</span> <span style="color: #8B008B; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> DecisionTreeRegressor
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> export_graphviz
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> StandardScaler, OneHotEncoder
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.compose</span> <span style="color: #8B008B; font-weight: bold">import</span> ColumnTransformer
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">IPython.display</span> <span style="color: #8B008B; font-weight: bold">import</span> Image
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">pydot</span> <span style="color: #8B008B; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> BaggingClassifier
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span>
|
||||
|
||||
<span style="color: #228B22"># Where to save the figures and data files</span>
|
||||
PROJECT_ROOT_DIR = <span style="color: #CD5555">"Results"</span>
|
||||
FIGURE_ID = <span style="color: #CD5555">"Results/FigureFiles"</span>
|
||||
DATA_ID = <span style="color: #CD5555">"DataFiles/"</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #8B008B">not</span> os.path.exists(PROJECT_ROOT_DIR):
|
||||
os.mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #8B008B">not</span> os.path.exists(FIGURE_ID):
|
||||
os.makedirs(FIGURE_ID)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #8B008B">not</span> os.path.exists(DATA_ID):
|
||||
os.makedirs(DATA_ID)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">image_path</span>(fig_id):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> os.path.join(FIGURE_ID, fig_id)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">data_path</span>(dat_id):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> os.path.join(DATA_ID, dat_id)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">save_fig</span>(fig_id):
|
||||
plt.savefig(image_path(fig_id) + <span style="color: #CD5555">".png"</span>, <span style="color: #658b00">format</span>=<span style="color: #CD5555">'png'</span>)
|
||||
|
||||
<span style="color: #228B22"># Load the cancer data</span>
|
||||
cancer = load_breast_cancer()
|
||||
|
||||
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=<span style="color: #B452CD">0</span>)
|
||||
<span style="color: #658b00">print</span>(X_train.shape)
|
||||
<span style="color: #658b00">print</span>(X_test.shape)
|
||||
<span style="color: #228B22">#Scale the data</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> StandardScaler
|
||||
scaler = StandardScaler()
|
||||
scaler.fit(X_train)
|
||||
X_train_scaled = scaler.transform(X_train)
|
||||
X_test_scaled = scaler.transform(X_test)
|
||||
<span style="color: #228B22">#define methods</span>
|
||||
<span style="color: #228B22"># Logistic Regression</span>
|
||||
logreg.fit(X_train_scaled, y_train)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test set accuracy Logistic Regression with scaled data: {:.2f}"</span>.format(logreg.score(X_test_scaled,y_test)))
|
||||
<span style="color: #228B22"># Support Vector Machine</span>
|
||||
svm.fit(X_train_scaled, y_train)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test set accuracy SVM with scaled data: {:.2f}"</span>.format(logreg.score(X_test_scaled,y_test)))
|
||||
<span style="color: #228B22"># Decision Trees</span>
|
||||
deep_tree_clf.fit(X_train_scaled, y_train)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test set accuracy with Decision Trees and scaled data: {:.2f}"</span>.format(deep_tree_clf.score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> LabelEncoder
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> cross_validate
|
||||
<span style="color: #228B22"># Data set not specificied</span>
|
||||
<span style="color: #228B22">#Instantiate the model with 500 trees and entropy as splitting criteria</span>
|
||||
Random_Forest_model = RandomForestClassifier(n_estimators=<span style="color: #B452CD">500</span>,criterion=<span style="color: #CD5555">"entropy"</span>)
|
||||
Random_Forest_model.fit(X_train_scaled, y_train)
|
||||
<span style="color: #228B22">#Cross validation</span>
|
||||
accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=<span style="color: #B452CD">10</span>)[<span style="color: #CD5555">'test_score'</span>]
|
||||
<span style="color: #658b00">print</span>(accuracy)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test set accuracy with Random Forests and scaled data: {:.2f}"</span>.format(Random_Forest_model.score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">scikitplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">skplt</span>
|
||||
y_pred = Random_Forest_model.predict(X_test_scaled)
|
||||
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=<span style="color: #8B008B; font-weight: bold">True</span>)
|
||||
plt.show()
|
||||
y_probas = Random_Forest_model.predict_proba(X_test_scaled)
|
||||
skplt.metrics.plot_roc(y_test, y_probas)
|
||||
plt.show()
|
||||
skplt.metrics.plot_cumulative_gain(y_test, y_probas)
|
||||
plt.show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="boosting-a-bird-s-eye-view">Boosting, a Bird's Eye View </h2>
|
||||
|
||||
|
||||
@@ -141,6 +141,10 @@ div.toc p,a {
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
||||
('Brief code reminder from last wekk',
|
||||
2,
|
||||
None,
|
||||
'brief-code-reminder-from-last-wekk'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -270,6 +274,122 @@ MathJax.Hub.Config({
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="brief-code-reminder-from-last-wekk">Brief code reminder from last wekk </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># Common imports</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> Image
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">pydot</span> <span style="color: #008000; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeRegressor
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> export_graphviz
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler, OneHotEncoder
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.compose</span> <span style="color: #008000; font-weight: bold">import</span> ColumnTransformer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> Image
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">pydot</span> <span style="color: #008000; font-weight: bold">import</span> graph_from_dot_data
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> BaggingClassifier
|
||||
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Where to save the figures and data files</span>
|
||||
PROJECT_ROOT_DIR <span style="color: #666666">=</span> <span style="color: #BA2121">"Results"</span>
|
||||
FIGURE_ID <span style="color: #666666">=</span> <span style="color: #BA2121">"Results/FigureFiles"</span>
|
||||
DATA_ID <span style="color: #666666">=</span> <span style="color: #BA2121">"DataFiles/"</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(PROJECT_ROOT_DIR):
|
||||
os<span style="color: #666666">.</span>mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(FIGURE_ID):
|
||||
os<span style="color: #666666">.</span>makedirs(FIGURE_ID)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(DATA_ID):
|
||||
os<span style="color: #666666">.</span>makedirs(DATA_ID)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">image_path</span>(fig_id):
|
||||
<span style="color: #008000; font-weight: bold">return</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(FIGURE_ID, fig_id)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">data_path</span>(dat_id):
|
||||
<span style="color: #008000; font-weight: bold">return</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(DATA_ID, dat_id)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">save_fig</span>(fig_id):
|
||||
plt<span style="color: #666666">.</span>savefig(image_path(fig_id) <span style="color: #666666">+</span> <span style="color: #BA2121">".png"</span>, <span style="color: #008000">format</span><span style="color: #666666">=</span><span style="color: #BA2121">'png'</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Load the cancer data</span>
|
||||
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
||||
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
||||
<span style="color: #008000">print</span>(X_train<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #008000">print</span>(X_test<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic">#Scale the data</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
scaler<span style="color: #666666">.</span>fit(X_train)
|
||||
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
||||
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
||||
<span style="color: #408080; font-style: italic">#define methods</span>
|
||||
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
||||
logreg<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy Logistic Regression with scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Support Vector Machine</span>
|
||||
svm<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy SVM with scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Decision Trees</span>
|
||||
deep_tree_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Decision Trees and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(deep_tree_clf<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> LabelEncoder
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
|
||||
<span style="color: #408080; font-style: italic"># Data set not specificied</span>
|
||||
<span style="color: #408080; font-style: italic">#Instantiate the model with 500 trees and entropy as splitting criteria</span>
|
||||
Random_Forest_model <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=500</span>,criterion<span style="color: #666666">=</span><span style="color: #BA2121">"entropy"</span>)
|
||||
Random_Forest_model<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #408080; font-style: italic">#Cross validation</span>
|
||||
accuracy <span style="color: #666666">=</span> cross_validate(Random_Forest_model,X_test_scaled,y_test,cv<span style="color: #666666">=10</span>)[<span style="color: #BA2121">'test_score'</span>]
|
||||
<span style="color: #008000">print</span>(accuracy)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Random Forests and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(Random_Forest_model<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
y_pred <span style="color: #666666">=</span> Random_Forest_model<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
y_probas <span style="color: #666666">=</span> Random_Forest_model<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="boosting-a-bird-s-eye-view">Boosting, a Bird's Eye View </h2>
|
||||
|
||||
|
||||
Binary file not shown.
@@ -2,7 +2,7 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e77aced1",
|
||||
"id": "06bf33a6",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -14,7 +14,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2cd56872",
|
||||
"id": "b1777bc2",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -29,7 +29,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e5d48181",
|
||||
"id": "f97929b1",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -53,7 +53,121 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b3df1935",
|
||||
"id": "da2cba17",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"## Brief code reminder from last wekk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "1054a91d",
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"# Common imports\n",
|
||||
"from IPython.display import Image \n",
|
||||
"from pydot import graph_from_dot_data\n",
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from sklearn.tree import DecisionTreeClassifier\n",
|
||||
"from sklearn.tree import DecisionTreeRegressor\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.tree import export_graphviz\n",
|
||||
"from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
|
||||
"from sklearn.compose import ColumnTransformer\n",
|
||||
"from IPython.display import Image \n",
|
||||
"from pydot import graph_from_dot_data\n",
|
||||
"from sklearn.datasets import load_breast_cancer\n",
|
||||
"from sklearn.svm import SVC\n",
|
||||
"from sklearn.linear_model import LogisticRegression\n",
|
||||
"from sklearn.ensemble import BaggingClassifier\n",
|
||||
"\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Where to save the figures and data files\n",
|
||||
"PROJECT_ROOT_DIR = \"Results\"\n",
|
||||
"FIGURE_ID = \"Results/FigureFiles\"\n",
|
||||
"DATA_ID = \"DataFiles/\"\n",
|
||||
"\n",
|
||||
"if not os.path.exists(PROJECT_ROOT_DIR):\n",
|
||||
" os.mkdir(PROJECT_ROOT_DIR)\n",
|
||||
"\n",
|
||||
"if not os.path.exists(FIGURE_ID):\n",
|
||||
" os.makedirs(FIGURE_ID)\n",
|
||||
"\n",
|
||||
"if not os.path.exists(DATA_ID):\n",
|
||||
" os.makedirs(DATA_ID)\n",
|
||||
"\n",
|
||||
"def image_path(fig_id):\n",
|
||||
" return os.path.join(FIGURE_ID, fig_id)\n",
|
||||
"\n",
|
||||
"def data_path(dat_id):\n",
|
||||
" return os.path.join(DATA_ID, dat_id)\n",
|
||||
"\n",
|
||||
"def save_fig(fig_id):\n",
|
||||
" plt.savefig(image_path(fig_id) + \".png\", format='png')\n",
|
||||
"\n",
|
||||
"# Load the cancer data\n",
|
||||
"cancer = load_breast_cancer()\n",
|
||||
"\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)\n",
|
||||
"print(X_train.shape)\n",
|
||||
"print(X_test.shape)\n",
|
||||
"#Scale the data\n",
|
||||
"from sklearn.preprocessing import StandardScaler\n",
|
||||
"scaler = StandardScaler()\n",
|
||||
"scaler.fit(X_train)\n",
|
||||
"X_train_scaled = scaler.transform(X_train)\n",
|
||||
"X_test_scaled = scaler.transform(X_test)\n",
|
||||
"#define methods\n",
|
||||
"# Logistic Regression\n",
|
||||
"logreg.fit(X_train_scaled, y_train)\n",
|
||||
"print(\"Test set accuracy Logistic Regression with scaled data: {:.2f}\".format(logreg.score(X_test_scaled,y_test)))\n",
|
||||
"# Support Vector Machine\n",
|
||||
"svm.fit(X_train_scaled, y_train)\n",
|
||||
"print(\"Test set accuracy SVM with scaled data: {:.2f}\".format(logreg.score(X_test_scaled,y_test)))\n",
|
||||
"# Decision Trees\n",
|
||||
"deep_tree_clf.fit(X_train_scaled, y_train)\n",
|
||||
"print(\"Test set accuracy with Decision Trees and scaled data: {:.2f}\".format(deep_tree_clf.score(X_test_scaled,y_test)))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"from sklearn.ensemble import RandomForestClassifier\n",
|
||||
"from sklearn.preprocessing import LabelEncoder\n",
|
||||
"from sklearn.model_selection import cross_validate\n",
|
||||
"# Data set not specificied\n",
|
||||
"#Instantiate the model with 500 trees and entropy as splitting criteria\n",
|
||||
"Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion=\"entropy\")\n",
|
||||
"Random_Forest_model.fit(X_train_scaled, y_train)\n",
|
||||
"#Cross validation\n",
|
||||
"accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score']\n",
|
||||
"print(accuracy)\n",
|
||||
"print(\"Test set accuracy with Random Forests and scaled data: {:.2f}\".format(Random_Forest_model.score(X_test_scaled,y_test)))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"import scikitplot as skplt\n",
|
||||
"y_pred = Random_Forest_model.predict(X_test_scaled)\n",
|
||||
"skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)\n",
|
||||
"plt.show()\n",
|
||||
"y_probas = Random_Forest_model.predict_proba(X_test_scaled)\n",
|
||||
"skplt.metrics.plot_roc(y_test, y_probas)\n",
|
||||
"plt.show()\n",
|
||||
"skplt.metrics.plot_cumulative_gain(y_test, y_probas)\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b647351f",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -73,7 +187,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "dd736cd1",
|
||||
"id": "e7dbffcd",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -87,7 +201,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "0c0b24c0",
|
||||
"id": "494a82af",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -99,7 +213,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4137e955",
|
||||
"id": "d27f68f8",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -116,7 +230,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "29d36032",
|
||||
"id": "4dabbeb4",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -128,7 +242,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "66757693",
|
||||
"id": "7937173f",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -142,7 +256,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ea578d5d",
|
||||
"id": "8872158c",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -154,7 +268,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a7db9269",
|
||||
"id": "1e5b19b8",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -167,7 +281,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1ccbd1f7",
|
||||
"id": "9c2e02d1",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -179,7 +293,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "eee1c02b",
|
||||
"id": "4244f14f",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -189,7 +303,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "59947382",
|
||||
"id": "91147e5f",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -217,7 +331,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "747706cf",
|
||||
"id": "af34f35f",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -233,7 +347,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "227d374a",
|
||||
"id": "cc2ea75e",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -245,7 +359,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8d5aef3c",
|
||||
"id": "5a5b4d62",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -256,7 +370,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5eefc213",
|
||||
"id": "daddba87",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -268,7 +382,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b172711a",
|
||||
"id": "2a6e0f8d",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -278,7 +392,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f04d3de1",
|
||||
"id": "f39ea127",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -290,7 +404,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3cb7e6dc",
|
||||
"id": "62a2184d",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -300,7 +414,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ce8ef5fc",
|
||||
"id": "a2781707",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -312,7 +426,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "05cf81cc",
|
||||
"id": "95c004bf",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -322,7 +436,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5fd7adfb",
|
||||
"id": "4694fd52",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -334,7 +448,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4fe5b6a2",
|
||||
"id": "06aea446",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -348,7 +462,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3d7d003d",
|
||||
"id": "20d085e6",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -364,7 +478,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d840b5f6",
|
||||
"id": "acc85c07",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -376,7 +490,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3ed8a69e",
|
||||
"id": "16f291d8",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -392,7 +506,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d24611b1",
|
||||
"id": "c3a7df55",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -404,7 +518,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "54adf710",
|
||||
"id": "9104a175",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -414,7 +528,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "63861b70",
|
||||
"id": "86ddaaa4",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -426,7 +540,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f078b4c8",
|
||||
"id": "ec36807d",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -438,7 +552,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "0b99a2d9",
|
||||
"id": "f102ce80",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -450,7 +564,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "bafb54a6",
|
||||
"id": "d7b0991f",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -461,7 +575,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "13403651",
|
||||
"id": "9c1752b5",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -473,7 +587,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "fa522c5a",
|
||||
"id": "7a9e3ab4",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -484,7 +598,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3c445ac4",
|
||||
"id": "6fb6c431",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -496,7 +610,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "142fa953",
|
||||
"id": "46ccbf78",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -506,7 +620,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ef2ba69c",
|
||||
"id": "cc303451",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -518,7 +632,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e6a323c0",
|
||||
"id": "b1c2be38",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -530,7 +644,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f1beb956",
|
||||
"id": "e45307bf",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -542,7 +656,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a8f78b87",
|
||||
"id": "edef26a5",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -554,7 +668,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1b0175d5",
|
||||
"id": "ef3f9795",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -564,7 +678,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "810e97f4",
|
||||
"id": "6772b2a5",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -576,7 +690,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5c54da4e",
|
||||
"id": "645ab1ac",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -586,7 +700,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8e1b0ba4",
|
||||
"id": "36afb932",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -598,7 +712,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b9e45cf8",
|
||||
"id": "a0e70734",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -608,7 +722,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8fc9fc75",
|
||||
"id": "a20d1c75",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -620,7 +734,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "24971ffa",
|
||||
"id": "415dcee2",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -630,7 +744,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "03d87d5d",
|
||||
"id": "18c44408",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -642,7 +756,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2727e0b5",
|
||||
"id": "2bf44e31",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -652,7 +766,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b106a65b",
|
||||
"id": "3cfed10d",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -664,7 +778,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "973dce9f",
|
||||
"id": "84f4d3f8",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -684,7 +798,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1d09daa2",
|
||||
"id": "4a886572",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -696,7 +810,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "301e4d68",
|
||||
"id": "e2da363e",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -706,7 +820,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "47876b6b",
|
||||
"id": "aec10b1a",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -722,7 +836,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "49f144bf",
|
||||
"id": "1814cd1d",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -734,7 +848,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a88c7e04",
|
||||
"id": "72b3a813",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -762,7 +876,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "77ebd598",
|
||||
"id": "7ec1c900",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -774,8 +888,8 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "135d466d",
|
||||
"execution_count": 2,
|
||||
"id": "341c3079",
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
@@ -807,7 +921,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "85cad394",
|
||||
"id": "ddaaaa66",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -825,7 +939,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "21ba0ee8",
|
||||
"id": "97b3c420",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -838,7 +952,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "7e418050",
|
||||
"id": "3e0cffa8",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -850,7 +964,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "96361e67",
|
||||
"id": "49e25f7c",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -860,7 +974,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e27ab8f2",
|
||||
"id": "f4eda3ea",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -872,7 +986,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a11891ae",
|
||||
"id": "6d9e5f3b",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -882,7 +996,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d9af023c",
|
||||
"id": "f924c7ad",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -894,7 +1008,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c89dbb5f",
|
||||
"id": "78272ab5",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -907,7 +1021,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "91a28699",
|
||||
"id": "0c5911f7",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -919,7 +1033,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "64077f83",
|
||||
"id": "476e645a",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -931,7 +1045,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5368f87c",
|
||||
"id": "0f989a46",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -943,7 +1057,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6aad78c8",
|
||||
"id": "b2668d41",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -953,7 +1067,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "bbd1f155",
|
||||
"id": "7f5965b6",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -965,7 +1079,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4302a358",
|
||||
"id": "32e6877d",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -975,7 +1089,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8e709038",
|
||||
"id": "61785a3c",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -991,7 +1105,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e2ce0ba3",
|
||||
"id": "df29a69e",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -1003,7 +1117,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "64d4c185",
|
||||
"id": "d8b61f23",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -1024,7 +1138,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4914feef",
|
||||
"id": "d067eeac",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -1034,16 +1148,14 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "81f5d24e",
|
||||
"execution_count": 3,
|
||||
"id": "2c80d8e8",
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import numpy as np\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
@@ -1089,7 +1201,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3ba6c8ed",
|
||||
"id": "4128dc16",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -1099,8 +1211,8 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "61116967",
|
||||
"execution_count": 4,
|
||||
"id": "ecc2c3bf",
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
@@ -1151,7 +1263,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9b4f86a3",
|
||||
"id": "7aa38b8c",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -1174,7 +1286,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3f326f1d",
|
||||
"id": "6cf5f0de",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -1184,8 +1296,8 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "1e12633b",
|
||||
"execution_count": 5,
|
||||
"id": "3d356194",
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
@@ -1237,7 +1349,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "23f47790",
|
||||
"id": "ed420a20",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -1249,8 +1361,8 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "e4014a85",
|
||||
"execution_count": 6,
|
||||
"id": "4de8cf9f",
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
|
||||
@@ -18,6 +18,105 @@ o Add material about AdaBoost and Gradient boosting
|
||||
!eblock
|
||||
|
||||
|
||||
!split
|
||||
===== Brief code reminder from last wekk =====
|
||||
|
||||
!bc pycod
|
||||
# Common imports
|
||||
from IPython.display import Image
|
||||
from pydot import graph_from_dot_data
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.tree import DecisionTreeRegressor
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.tree import export_graphviz
|
||||
from sklearn.preprocessing import StandardScaler, OneHotEncoder
|
||||
from sklearn.compose import ColumnTransformer
|
||||
from IPython.display import Image
|
||||
from pydot import graph_from_dot_data
|
||||
from sklearn.datasets import load_breast_cancer
|
||||
from sklearn.svm import SVC
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.ensemble import BaggingClassifier
|
||||
|
||||
import os
|
||||
|
||||
# Where to save the figures and data files
|
||||
PROJECT_ROOT_DIR = "Results"
|
||||
FIGURE_ID = "Results/FigureFiles"
|
||||
DATA_ID = "DataFiles/"
|
||||
|
||||
if not os.path.exists(PROJECT_ROOT_DIR):
|
||||
os.mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
if not os.path.exists(FIGURE_ID):
|
||||
os.makedirs(FIGURE_ID)
|
||||
|
||||
if not os.path.exists(DATA_ID):
|
||||
os.makedirs(DATA_ID)
|
||||
|
||||
def image_path(fig_id):
|
||||
return os.path.join(FIGURE_ID, fig_id)
|
||||
|
||||
def data_path(dat_id):
|
||||
return os.path.join(DATA_ID, dat_id)
|
||||
|
||||
def save_fig(fig_id):
|
||||
plt.savefig(image_path(fig_id) + ".png", format='png')
|
||||
|
||||
# Load the cancer data
|
||||
cancer = load_breast_cancer()
|
||||
|
||||
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
|
||||
print(X_train.shape)
|
||||
print(X_test.shape)
|
||||
#Scale the data
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
scaler = StandardScaler()
|
||||
scaler.fit(X_train)
|
||||
X_train_scaled = scaler.transform(X_train)
|
||||
X_test_scaled = scaler.transform(X_test)
|
||||
#define methods
|
||||
# Logistic Regression
|
||||
logreg.fit(X_train_scaled, y_train)
|
||||
print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
|
||||
# Support Vector Machine
|
||||
svm.fit(X_train_scaled, y_train)
|
||||
print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
|
||||
# Decision Trees
|
||||
deep_tree_clf.fit(X_train_scaled, y_train)
|
||||
print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
from sklearn.ensemble import RandomForestClassifier
|
||||
from sklearn.preprocessing import LabelEncoder
|
||||
from sklearn.model_selection import cross_validate
|
||||
# Data set not specificied
|
||||
#Instantiate the model with 500 trees and entropy as splitting criteria
|
||||
Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy")
|
||||
Random_Forest_model.fit(X_train_scaled, y_train)
|
||||
#Cross validation
|
||||
accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score']
|
||||
print(accuracy)
|
||||
print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
import scikitplot as skplt
|
||||
y_pred = Random_Forest_model.predict(X_test_scaled)
|
||||
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
|
||||
plt.show()
|
||||
y_probas = Random_Forest_model.predict_proba(X_test_scaled)
|
||||
skplt.metrics.plot_roc(y_test, y_probas)
|
||||
plt.show()
|
||||
skplt.metrics.plot_cumulative_gain(y_test, y_probas)
|
||||
plt.show()
|
||||
|
||||
!ec
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
!split
|
||||
@@ -657,3 +756,4 @@ plt.show()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user