diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs000.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs000.html index d3af51f30..42e102612 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs000.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs000.html @@ -100,20 +100,25 @@ Automatically generated HTML file from DocOnce source '___sec36'), ('Now Bagging', 2, None, '___sec37'), ('Random forests', 2, None, '___sec38'), - ('A simple scikit-learn example', 2, None, '___sec39'), - ('Then random forests', 2, None, '___sec40'), - ('Feature Importance', 2, None, '___sec41'), - ("Boosting, a Bird'e Eye", 2, None, '___sec42'), + ('Random Forest Algorithm', 2, None, '___sec39'), + ('A simple scikit-learn example', 2, None, '___sec40'), + ('Then random forests', 2, None, '___sec41'), + ('Feature Importance', 2, None, '___sec42'), + ("Boosting, a Bird'e Eye", 2, None, '___sec43'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec43'), - ('Basic Steps of AdaBoost', 2, None, '___sec44'), - ('AdaBoost Examples', 2, None, '___sec45'), - ('Gradient boosting: Basic Algorithm', 2, None, '___sec46'), - ('Gradient Boosting, Examples', 2, None, '___sec47'), - ('Gradient Boots with Early Stopping', 2, None, '___sec48'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec49')]} + '___sec44'), + ('Basic Steps of AdaBoost', 2, None, '___sec45'), + ('Figure to Illustrate the Iterative Classification Process', + 2, + None, + '___sec46'), + ('AdaBoost Examples', 2, None, '___sec47'), + ('Gradient boosting: Basic Algorithm', 2, None, '___sec48'), + ('Gradient Boosting, Examples', 2, None, '___sec49'), + ('Gradient Boots with Early Stopping', 2, None, '___sec50'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec51')]} end of tocinfo -->
@@ -190,17 +195,19 @@ MathJax.Hub.Config({
@@ -1771,7 +1777,7 @@ accuracy = cross_validate(Random_Forest_model,X,Y,cv=Then random forests
+
@@ -1794,7 +1800,7 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
Example will be added here.
@@ -1802,7 +1808,7 @@ Example will be added here.
The basic idea is to combine weak classifiers in order to create a good
@@ -1819,7 +1825,7 @@ them with a factor.
The algorithm here is rather straightforward. Assume that our weak
@@ -1843,7 +1849,7 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
With the above definitions we are now ready to set up the algorithm for AdaBoost.
@@ -1858,11 +1864,42 @@ $$
\mathrm{err}=\frac{\sum_{i=0}^{n-1}w_iI(y_i\ne G(\boldsymbol{X}_{i*})}{\sum_{i=0}^{n-1}w_i},
$$
+
+For the iterations with \( m \le 2 \) the weights are modified
+individually at each steps. The obersvations which were misclassified
+at iteration \( m-1 \) have a weight which is larger than those which were
+classified properly. As this proceeds, the observations which were
+difficult to classifiy correctly are given a larger influence. Each
+new classificatio step \( m \) is then forced to concentrate on those
+observations that are missed in the previous iterations.
+Using Scikit-Learn it is easy to appply the adaptive boosting algorithm, as done here.
+
@@ -1902,12 +1939,12 @@ plt.show()
@@ -1998,7 +2035,7 @@ plt.show()
@@ -2063,7 +2100,7 @@ error_going_up = 0
+
@@ -1750,7 +1761,7 @@ accuracy = cross_validate(Random_Forest_model,X,Y,cv=Then random forests
+
@@ -1772,7 +1783,7 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
Example will be added here.
@@ -1780,7 +1791,7 @@ Example will be added here.
The basic idea is to combine weak classifiers in order to create a good
@@ -1797,7 +1808,7 @@ them with a factor.
The algorithm here is rather straightforward. Assume that our weak
@@ -1819,7 +1830,7 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
With the above definitions we are now ready to set up the algorithm for AdaBoost.
@@ -1834,10 +1845,41 @@ $$
\mathrm{err}=\frac{\sum_{i=0}^{n-1}w_iI(y_i\ne G(\boldsymbol{X}_{i*})}{\sum_{i=0}^{n-1}w_i},
$$
+
+
+
+Using Scikit-Learn it is easy to appply the adaptive boosting algorithm, as done here.
+
@@ -1876,12 +1918,12 @@ plt.show()
@@ -1971,7 +2013,7 @@ plt.show()
@@ -2035,7 +2077,7 @@ error_going_up = 0
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees.html b/doc/pub/DecisionTrees/html/DecisionTrees.html
index 50175bd87..c0ce80a23 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees.html
@@ -125,20 +125,25 @@ div { text-align: justify; text-justify: inter-word; }
'___sec36'),
('Now Bagging', 2, None, '___sec37'),
('Random forests', 2, None, '___sec38'),
- ('A simple scikit-learn example', 2, None, '___sec39'),
- ('Then random forests', 2, None, '___sec40'),
- ('Feature Importance', 2, None, '___sec41'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec42'),
+ ('Random Forest Algorithm', 2, None, '___sec39'),
+ ('A simple scikit-learn example', 2, None, '___sec40'),
+ ('Then random forests', 2, None, '___sec41'),
+ ('Feature Importance', 2, None, '___sec42'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec43'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec43'),
- ('Basic Steps of AdaBoost', 2, None, '___sec44'),
- ('AdaBoost Examples', 2, None, '___sec45'),
- ('Gradient boosting: Basic Algorithm', 2, None, '___sec46'),
- ('Gradient Boosting, Examples', 2, None, '___sec47'),
- ('Gradient Boots with Early Stopping', 2, None, '___sec48'),
- ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec49')]}
+ '___sec44'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec45'),
+ ('Figure to Illustrate the Iterative Classification Process',
+ 2,
+ None,
+ '___sec46'),
+ ('AdaBoost Examples', 2, None, '___sec47'),
+ ('Gradient boosting: Basic Algorithm', 2, None, '___sec48'),
+ ('Gradient Boosting, Examples', 2, None, '___sec49'),
+ ('Gradient Boots with Early Stopping', 2, None, '___sec50'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec51')]}
end of tocinfo -->
+
@@ -1755,7 +1766,7 @@ accuracy = cross_validate(Random_Forest_mode
@@ -1777,7 +1788,7 @@ np.sum(y_pred =
Example will be added here.
@@ -1785,7 +1796,7 @@ Example will be added here.
The basic idea is to combine weak classifiers in order to create a good
@@ -1802,7 +1813,7 @@ them with a factor.
The algorithm here is rather straightforward. Assume that our weak
@@ -1824,7 +1835,7 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
With the above definitions we are now ready to set up the algorithm for AdaBoost.
@@ -1839,10 +1850,41 @@ $$
\mathrm{err}=\frac{\sum_{i=0}^{n-1}w_iI(y_i\ne G(\boldsymbol{X}_{i*})}{\sum_{i=0}^{n-1}w_i},
$$
+
+
+
+Using Scikit-Learn it is easy to appply the adaptive boosting algorithm, as done here.
+
@@ -1881,12 +1923,12 @@ plt.show()
@@ -1976,7 +2018,7 @@ plt.show()
@@ -2040,7 +2082,7 @@ error_going_up = XGBoost: Extreme Gradient Boosting
+
diff --git a/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb b/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
index e84a1c377..7cf60d741 100644
--- a/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
+++ b/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
@@ -1762,6 +1762,10 @@
"not lead to a substantial reduction in variance over a single tree in\n",
"this setting.\n",
"\n",
+ "\n",
+ "## Random Forest Algorithm\n",
+ "The algorithm described here can be applied to both classification and regression problems.\n",
+ "\n",
"## A simple scikit-learn example"
]
},
@@ -1894,7 +1898,33 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "## AdaBoost Examples"
+ "1. Then we start looping over all attempts at classifying, namely we start an iterative process for $m=1:M$, where $M$ is the final number of classifications. Our given classifier could for example be a plain decision tree.\n",
+ "\n",
+ "a. Fit thus a given classifier to the training using the weights $w_i$.\n",
+ "\n",
+ "b. Compute then $\\mathrm{err}$ and figure out which events are classified properly and which are classified wrongly.\n",
+ "\n",
+ "c. Define a quantity $\\alpha_{m} = \\log{(1-\\mathrm{err})/\\mathrm{err}}\n",
+ "\n",
+ "d. Set the new weights to $w_i = w_i\\times \\exp{(\\alpha_m I(y_i\\ne G(\\boldsymbol{X}_{i*})}.\n",
+ "\n",
+ "\n",
+ "5. Compute the new classifier $G(\\boldsymbol{X})= \\sum_{i=0}^{n-1}\\alpha_m I(y_i\\ne G(\\boldsymbol{X}_{i*}).\n",
+ "\n",
+ "For the iterations with $m \\le 2$ the weights are modified\n",
+ "individually at each steps. The obersvations which were misclassified\n",
+ "at iteration $m-1$ have a weight which is larger than those which were\n",
+ "classified properly. As this proceeds, the observations which were\n",
+ "difficult to classifiy correctly are given a larger influence. Each\n",
+ "new classificatio step $m$ is then forced to concentrate on those\n",
+ "observations that are missed in the previous iterations.\n",
+ "\n",
+ "## Figure to Illustrate the Iterative Classification Process\n",
+ "\n",
+ "\n",
+ "## AdaBoost Examples\n",
+ "\n",
+ "Using **Scikit-Learn** it is easy to appply the adaptive boosting algorithm, as done here."
]
},
{
diff --git a/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz b/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz
index f87e98931..39e57db23 100644
Binary files a/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz and b/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz differ
diff --git a/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf b/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf
index 53ae56b4b..b20336b32 100644
Binary files a/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf and b/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf differ
diff --git a/doc/src/DecisionTrees/DecisionTrees.do.txt b/doc/src/DecisionTrees/DecisionTrees.do.txt
index 787ba6bb8..dc02103a0 100644
--- a/doc/src/DecisionTrees/DecisionTrees.do.txt
+++ b/doc/src/DecisionTrees/DecisionTrees.do.txt
@@ -1420,6 +1420,11 @@ uncorrelated quanti- ties. In particular, this means that bagging will
not lead to a substantial reduction in variance over a single tree in
this setting.
+
+!split
+===== Random Forest Algorithm =====
+The algorithm described here can be applied to both classification and regression problems.
+
!split
===== A simple scikit-learn example =====
!bc pycod
@@ -1509,11 +1514,30 @@ o We rewrite the misclassification error as
\mathrm{err}=\frac{\sum_{i=0}^{n-1}w_iI(y_i\ne G(\bm{X}_{i*})}{\sum_{i=0}^{n-1}w_i},
\]
!et
+o Then we start looping over all attempts at classifying, namely we start an iterative process for $m=1:M$, where $M$ is the final number of classifications. Our given classifier could for example be a plain decision tree.
+ o Fit thus a given classifier to the training using the weights $w_i$.
+ o Compute then $\mathrm{err}$ and figure out which events are classified properly and which are classified wrongly.
+ o Define a quantity $\alpha_{m} = \log{(1-\mathrm{err})/\mathrm{err}}
+ o Set the new weights to $w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(\bm{X}_{i*})}.
+o Compute the new classifier $G(\bm{X})= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(\bm{X}_{i*}).
+For the iterations with $m \le 2$ the weights are modified
+individually at each steps. The obersvations which were misclassified
+at iteration $m-1$ have a weight which is larger than those which were
+classified properly. As this proceeds, the observations which were
+difficult to classifiy correctly are given a larger influence. Each
+new classificatio step $m$ is then forced to concentrate on those
+observations that are missed in the previous iterations.
+
+!split
+===== Figure to Illustrate the Iterative Classification Process =====
!split
===== AdaBoost Examples =====
+
+Using _Scikit-Learn_ it is easy to appply the adaptive boosting algorithm, as done here.
+
!bc pycod
from sklearn.ensemble import AdaBoostClassifier
Then random forests
Feature Importance
+Feature Importance
Boosting, a Bird'e Eye
+Boosting, a Bird'e Eye
Adaptive boosting: AdaBoost, Basic Algorithm
+Adaptive boosting: AdaBoost, Basic Algorithm
Basic Steps of AdaBoost
+Basic Steps of AdaBoost
+
+
+
+
+AdaBoost Examples
+Figure to Illustrate the Iterative Classification Process
+AdaBoost Examples
+
+Gradient boosting: Basic Algorithm
+Gradient boosting: Basic Algorithm
Gradient Boosting, Examples
+Gradient Boosting, Examples
Gradient Boots with Early Stopping
+Gradient Boots with Early Stopping
XGBoost: Extreme Gradient Boosting
+XGBoost: Extreme Gradient Boosting
-A simple scikit-learn example
+Random Forest Algorithm
+The algorithm described here can be applied to both classification and regression problems.
+
+
+
+A simple scikit-learn example
Then random forests
-Feature Importance
+Feature Importance
-Boosting, a Bird'e Eye
+Boosting, a Bird'e Eye
-Adaptive boosting: AdaBoost, Basic Algorithm
+Adaptive boosting: AdaBoost, Basic Algorithm
-Basic Steps of AdaBoost
+Basic Steps of AdaBoost
+
+
+
-AdaBoost Examples
+Figure to Illustrate the Iterative Classification Process
+
+
+
+AdaBoost Examples
+
+
-Gradient boosting: Basic Algorithm
+Gradient boosting: Basic Algorithm
-Gradient Boosting, Examples
+Gradient Boosting, Examples
-Gradient Boots with Early Stopping
+Gradient Boots with Early Stopping
-XGBoost: Extreme Gradient Boosting
+XGBoost: Extreme Gradient Boosting
-A simple scikit-learn example
+Random Forest Algorithm
+The algorithm described here can be applied to both classification and regression problems.
+
+
+
+A simple scikit-learn example
-Then random forests
+Then random forests
-Feature Importance
+Feature Importance
-Boosting, a Bird'e Eye
+Boosting, a Bird'e Eye
-Adaptive boosting: AdaBoost, Basic Algorithm
+Adaptive boosting: AdaBoost, Basic Algorithm
-Basic Steps of AdaBoost
+Basic Steps of AdaBoost
+
+
+
-AdaBoost Examples
+Figure to Illustrate the Iterative Classification Process
+
+
+
+AdaBoost Examples
+
+
-Gradient boosting: Basic Algorithm
+Gradient boosting: Basic Algorithm
-Gradient Boosting, Examples
+Gradient Boosting, Examples
-Gradient Boots with Early Stopping
+Gradient Boots with Early Stopping
XGBoost: Extreme Gradient Boosting