-
@@ -277,7 +283,7 @@ MathJax.Hub.Config({
Two algorithms stand out in the set up of decision trees:
@@ -270,7 +278,7 @@ We discuss both algorithms with applications here. The popular library -Scikit-L
As stated above and seen in many of the examples discussed here about @@ -250,7 +256,7 @@ machine learning algorithms or just use one of them to construct forests and jun
-Random forests provide an improvement over bagged trees by way of a -small tweak that decorrelates the trees. -
-As in bagging, we build a -number of decision trees on bootstrapped training samples. But when -building these decision trees, each time a split in a tree is -considered, a random sample of \( m \) predictors is chosen as split -candidates from the full set of \( p \) predictors. The split is allowed to -use only one of those \( m \) predictors. + +
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
-
-A fresh sample of \( m \) predictors is
-taken at each split, and typically we choose
-$$
-m\approx \sqrt{p}.
-$$
+np.random.seed(2018)
-
-In building a random forest, at
-each split in the tree, the algorithm is not even allowed to consider
-a majority of the available predictors.
+n = 40
+n_boostraps = 100
+maxdegree = 14
-
-The reason for this is rather clever. Suppose that there is one very
-strong predictor in the data set, along with a number of other
-moderately strong predictors. Then in the collection of bagged
-variable importance random forest trees, most or all of the trees will
-use this strong predictor in the top split. Consequently, all of the
-bagged trees will look quite similar to each other. Hence the
-predictions from the bagged trees will be highly correlated.
-Unfortunately, averaging many highly correlated quantities does not
-lead to as large of a reduction in variance as averaging many
-uncorrelated quanti- ties. In particular, this means that bagging will
-not lead to a substantial reduction in variance over a single tree in
-this setting.
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = DecisionTreeRegressor(max_depth=5)
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
@@ -301,7 +323,7 @@ this setting.
-We will grow of forest of say \( M \) trees. +Random forests provide an improvement over bagged trees by way of a +small tweak that decorrelates the trees. -
+As in bagging, we build a +number of decision trees on bootstrapped training samples. But when +building these decision trees, each time a split in a tree is +considered, a random sample of \( m \) predictors is chosen as split +candidates from the full set of \( p \) predictors. The split is allowed to +use only one of those \( m \) predictors. -
+A fresh sample of \( m \) predictors is +taken at each split, and typically we choose -
+In building a random forest, at +each split in the tree, the algorithm is not even allowed to consider +a majority of the available predictors. -
+The reason for this is rather clever. Suppose that there is one very +strong predictor in the data set, along with a number of other +moderately strong predictors. Then in the collection of bagged +variable importance random forest trees, most or all of the trees will +use this strong predictor in the top split. Consequently, all of the +bagged trees will look quite similar to each other. Hence the +predictions from the bagged trees will be highly correlated. +Unfortunately, averaging many highly correlated quantities does not +lead to as large of a reduction in variance as averaging many +uncorrelated quanti- ties. In particular, this means that bagging will +not lead to a substantial reduction in variance over a single tree in +this setting. +
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs042.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs042.html index 4b3db8b2b..486a0c3b1 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs042.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs042.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
+We will grow of forest of say \( M \) trees. - -
import matplotlib.pyplot as plt
-import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn.datasets import load_breast_cancer
-from sklearn.svm import SVC
-from sklearn.linear_model import LogisticRegression
-from sklearn.tree import DecisionTreeClassifier
+
+- For \( m=1:M \) we
-# Load the data
-cancer = load_breast_cancer()
+
+ - Draw a bootstrap sample of from the training data organized in our \( \boldsymbol{X} \) matrix.
+ - We grow then a random forest tree \( T_m \) based on the bootstrapped data by repeating the steps outlined till we reach the maximum node size is reached
-X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-print(X_train.shape)
-print(X_test.shape)
-# Logistic Regression
-logreg = LogisticRegression(solver='lbfgs')
-logreg.fit(X_train, y_train)
-print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
-# Support vector machine
-svm = SVC(gamma='auto', C=100)
-svm.fit(X_train, y_train)
-print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
-# Decision Trees
-deep_tree_clf = DecisionTreeClassifier(max_depth=None)
-deep_tree_clf.fit(X_train, y_train)
-print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
-#now scale the data
-from sklearn.preprocessing import StandardScaler
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-# Logistic Regression
-logreg.fit(X_train_scaled, y_train)
-print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Support Vector Machine
-svm.fit(X_train_scaled, y_train)
-print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Decision Trees
-deep_tree_clf.fit(X_train_scaled, y_train)
-print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
+
+ - we select \( m \le p \) varibales at random from the \( p \) predictors/features
+ - pick the best split point among the \( m \) features using either the CART algorithm or the ID3 for classification and create a new node
+ - split the node into daughter nodes
+
+
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.preprocessing import LabelEncoder
-from sklearn.model_selection import cross_validate
-# Data set not specificied
-#Instantiate the model with 500 trees and entropy as splitting criteria
-Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy")
-Random_Forest_model.fit(X_train_scaled, y_train)
-#Cross validation
-accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score']
-print(accuracy)
-print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test)))
+- Output then the ensemble of trees \( \{T_m\}_1^{M} \) and make predictions for either a regression type of problem or a classification type of problem.
+
-
-import scikitplot as skplt
-y_pred = Random_Forest_model.predict(X_test_scaled)
-skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
-#<matplotlib.axes._subplots.AxesSubplot object at 0x7fe967d64490>
-plt.show()
-y_probas = Random_Forest_model.predict_proba(X_test_scaled)
-skplt.metrics.plot_roc(y_test, y_probas)
-plt.show()
-skplt.metrics.plot_cumulative_gain(y_test, y_probas)
-plt.show()
-
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs043.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs043.html index c8ef5cf7a..c4bc06499 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs043.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs043.html @@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source ('Final regressor code', 2, None, '___sec28'), ('Pros and cons of trees, pros', 2, None, '___sec29'), ('Disadvantages', 2, None, '___sec30'), - ('From a Single Tree to Many Trees, Meet the Jungle of Methods', + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', 2, None, '___sec31'), @@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source None, '___sec37'), ('Now Bagging', 2, None, '___sec38'), - ('Random forests', 2, None, '___sec39'), - ('Random Forest Algorithm', 2, None, '___sec40'), + ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), + ('Random forests', 2, None, '___sec40'), + ('Random Forest Algorithm', 2, None, '___sec41'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec41'), + '___sec42'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec42'), - ('Feature Importance', 2, None, '___sec43'), - ("Boosting, a Bird'e Eye", 2, None, '___sec44'), + '___sec43'), + ('Bootstrap with Random Forests Instead of a Single Tree', + 2, + None, + '___sec44'), + ("Boosting, a Bird'e Eye", 2, None, '___sec45'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec45'), - ('Basic Steps of AdaBoost', 2, None, '___sec46'), + '___sec46'), + ('Basic Steps of AdaBoost', 2, None, '___sec47'), ('Figure to Illustrate the Iterative Classification Process', 2, None, - '___sec47'), - ('AdaBoost Examples', 2, None, '___sec48'), - ('Gradient boosting: Basics', 2, None, '___sec49'), - ('Gradient Boosting, algorithm', 2, None, '___sec50'), - ('Gradient Boosting, Examples', 2, None, '___sec51'), - ('Gradient Boots with Early Stopping', 2, None, '___sec52'), - ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'), - ('Xgboost on the Cancer Data', 2, None, '___sec54')]} + '___sec48'), + ('AdaBoost Examples', 2, None, '___sec49'), + ('Gradient boosting: Basics', 2, None, '___sec50'), + ('Gradient Boosting, algorithm', 2, None, '___sec51'), + ('Gradient Boosting, Examples', 2, None, '___sec52'), + ('Gradient Boots with Early Stopping', 2, None, '___sec53'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), + ('Xgboost on the Cancer Data', 2, None, '___sec55')]} end of tocinfo --> @@ -195,7 +200,7 @@ MathJax.Hub.Config({
-
bag_clf = BaggingClassifier(
- DecisionTreeClassifier(splitter="random", max_leaf_nodes=16, random_state=42),
- n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1, random_state=42)
-+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+from sklearn.svm import SVC
+from sklearn.linear_model import LogisticRegression
+from sklearn.tree import DecisionTreeClassifier
+
+# Load the data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+# Logistic Regression
+logreg = LogisticRegression(solver='lbfgs')
+logreg.fit(X_train, y_train)
+print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
+# Support vector machine
+svm = SVC(gamma='auto', C=100)
+svm.fit(X_train, y_train)
+print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
+# Decision Trees
+deep_tree_clf = DecisionTreeClassifier(max_depth=None)
+deep_tree_clf.fit(X_train, y_train)
+print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+# Logistic Regression
+logreg.fit(X_train_scaled, y_train)
+print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Support Vector Machine
+svm.fit(X_train_scaled, y_train)
+print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Decision Trees
+deep_tree_clf.fit(X_train_scaled, y_train)
+print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
+
-
-bag_clf.fit(X_train, y_train)
-y_pred = bag_clf.predict(X_test)
from sklearn.ensemble import RandomForestClassifier
-rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)
-rnd_clf.fit(X_train, y_train)
-y_pred_rf = rnd_clf.predict(X_test)
-np.sum(y_pred == y_pred_rf) / len(y_pred)
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+# Data set not specificied
+#Instantiate the model with 500 trees and entropy as splitting criteria
+Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy")
+Random_Forest_model.fit(X_train_scaled, y_train)
+#Cross validation
+accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score']
+print(accuracy)
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test)))
+
+
+import scikitplot as skplt
+y_pred = Random_Forest_model.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = Random_Forest_model.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
@@ -279,7 +336,7 @@ np.sum(y_pred =
52
53
...
- 56
+ 57
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs044.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs044.html
index c863be7df..2401d68f0 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs044.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs044.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
('Final regressor code', 2, None, '___sec28'),
('Pros and cons of trees, pros', 2, None, '___sec29'),
('Disadvantages', 2, None, '___sec30'),
- ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
2,
None,
'___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
None,
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
- ('Random forests', 2, None, '___sec39'),
- ('Random Forest Algorithm', 2, None, '___sec40'),
+ ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+ ('Random forests', 2, None, '___sec40'),
+ ('Random Forest Algorithm', 2, None, '___sec41'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec41'),
+ '___sec42'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec42'),
- ('Feature Importance', 2, None, '___sec43'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+ '___sec43'),
+ ('Bootstrap with Random Forests Instead of a Single Tree',
+ 2,
+ None,
+ '___sec44'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec45'),
- ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+ '___sec46'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec47'),
('Figure to Illustrate the Iterative Classification Process',
2,
None,
- '___sec47'),
- ('AdaBoost Examples', 2, None, '___sec48'),
- ('Gradient boosting: Basics', 2, None, '___sec49'),
- ('Gradient Boosting, algorithm', 2, None, '___sec50'),
- ('Gradient Boosting, Examples', 2, None, '___sec51'),
- ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
- ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
- ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+ '___sec48'),
+ ('AdaBoost Examples', 2, None, '___sec49'),
+ ('Gradient boosting: Basics', 2, None, '___sec50'),
+ ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+ ('Gradient Boosting, Examples', 2, None, '___sec52'),
+ ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
end of tocinfo -->
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
Final regressor code
Pros and cons of trees, pros
Disadvantages
- From a Single Tree to Many Trees, Meet the Jungle of Methods
+ Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
An Overview of Ensemble Methods
Bagging
More bagging
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
Using the Voting Classifier
Please, not the moons again! Voting and Bagging
Now Bagging
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Feature Importance
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
- AdaBoost Examples
- Gradient boosting: Basics
- Gradient Boosting, algorithm
- Gradient Boosting, Examples
- Gradient Boots with Early Stopping
- XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Making our own Bagging with Bootstrap
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
+ Figure to Illustrate the Iterative Classification Process
+ AdaBoost Examples
+ Gradient boosting: Basics
+ Gradient Boosting, algorithm
+ Gradient Boosting, Examples
+ Gradient Boots with Early Stopping
+ XGBoost: Extreme Gradient Boosting
+ Xgboost on the Cancer Data
@@ -234,11 +240,25 @@ MathJax.Hub.Config({
-Feature Importance
-
+Compare Bagging on Trees with Random Forests
-Example will be added here.
+
+
bag_clf = BaggingClassifier(
+ DecisionTreeClassifier(splitter="random", max_leaf_nodes=16, random_state=42),
+ n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1, random_state=42)
+
+
+
+
+
bag_clf.fit(X_train, y_train)
+y_pred = bag_clf.predict(X_test)
+from sklearn.ensemble import RandomForestClassifier
+rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)
+rnd_clf.fit(X_train, y_train)
+y_pred_rf = rnd_clf.predict(X_test)
+np.sum(y_pred == y_pred_rf) / len(y_pred)
+
@@ -265,7 +285,7 @@ Example will be added here.
53
54
...
- 56
+ 57
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs045.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs045.html
index 94e51ba14..1431d4eb5 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs045.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs045.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
('Final regressor code', 2, None, '___sec28'),
('Pros and cons of trees, pros', 2, None, '___sec29'),
('Disadvantages', 2, None, '___sec30'),
- ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
2,
None,
'___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
None,
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
- ('Random forests', 2, None, '___sec39'),
- ('Random Forest Algorithm', 2, None, '___sec40'),
+ ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+ ('Random forests', 2, None, '___sec40'),
+ ('Random Forest Algorithm', 2, None, '___sec41'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec41'),
+ '___sec42'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec42'),
- ('Feature Importance', 2, None, '___sec43'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+ '___sec43'),
+ ('Bootstrap with Random Forests Instead of a Single Tree',
+ 2,
+ None,
+ '___sec44'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec45'),
- ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+ '___sec46'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec47'),
('Figure to Illustrate the Iterative Classification Process',
2,
None,
- '___sec47'),
- ('AdaBoost Examples', 2, None, '___sec48'),
- ('Gradient boosting: Basics', 2, None, '___sec49'),
- ('Gradient Boosting, algorithm', 2, None, '___sec50'),
- ('Gradient Boosting, Examples', 2, None, '___sec51'),
- ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
- ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
- ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+ '___sec48'),
+ ('AdaBoost Examples', 2, None, '___sec49'),
+ ('Gradient boosting: Basics', 2, None, '___sec50'),
+ ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+ ('Gradient Boosting, Examples', 2, None, '___sec52'),
+ ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
end of tocinfo -->
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
Final regressor code
Pros and cons of trees, pros
Disadvantages
- From a Single Tree to Many Trees, Meet the Jungle of Methods
+ Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
An Overview of Ensemble Methods
Bagging
More bagging
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
Using the Voting Classifier
Please, not the moons again! Voting and Bagging
Now Bagging
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Feature Importance
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
- AdaBoost Examples
- Gradient boosting: Basics
- Gradient Boosting, algorithm
- Gradient Boosting, Examples
- Gradient Boots with Early Stopping
- XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Making our own Bagging with Bootstrap
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
+ Figure to Illustrate the Iterative Classification Process
+ AdaBoost Examples
+ Gradient boosting: Basics
+ Gradient Boosting, algorithm
+ Gradient Boosting, Examples
+ Gradient Boots with Early Stopping
+ XGBoost: Extreme Gradient Boosting
+ Xgboost on the Cancer Data
@@ -234,20 +240,63 @@ MathJax.Hub.Config({
-Boosting, a Bird'e Eye
+Bootstrap with Random Forests Instead of a Single Tree
-The basic idea is to combine weak classifiers in order to create a good
-classifier. With a weak classifier we often intend a classifier which
-produces results which are only slightly better than we would get by
-random guesses.
-
-This is done by applying in an iterative way a weak (or a standard
-classifier like decision trees) to modify the data. In each iteration
-we emphasize those observations which are misclassified by weighting
-them with a factor.
+
+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.ensemble import RandomForestRegressor
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = RandomForestRegressor()
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
@@ -274,7 +323,7 @@ them with a factor.
54
55
...
- 56
+ 57
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs046.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs046.html
index 1eccd68a0..78f3f81a4 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs046.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs046.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
('Final regressor code', 2, None, '___sec28'),
('Pros and cons of trees, pros', 2, None, '___sec29'),
('Disadvantages', 2, None, '___sec30'),
- ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
2,
None,
'___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
None,
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
- ('Random forests', 2, None, '___sec39'),
- ('Random Forest Algorithm', 2, None, '___sec40'),
+ ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+ ('Random forests', 2, None, '___sec40'),
+ ('Random Forest Algorithm', 2, None, '___sec41'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec41'),
+ '___sec42'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec42'),
- ('Feature Importance', 2, None, '___sec43'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+ '___sec43'),
+ ('Bootstrap with Random Forests Instead of a Single Tree',
+ 2,
+ None,
+ '___sec44'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec45'),
- ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+ '___sec46'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec47'),
('Figure to Illustrate the Iterative Classification Process',
2,
None,
- '___sec47'),
- ('AdaBoost Examples', 2, None, '___sec48'),
- ('Gradient boosting: Basics', 2, None, '___sec49'),
- ('Gradient Boosting, algorithm', 2, None, '___sec50'),
- ('Gradient Boosting, Examples', 2, None, '___sec51'),
- ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
- ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
- ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+ '___sec48'),
+ ('AdaBoost Examples', 2, None, '___sec49'),
+ ('Gradient boosting: Basics', 2, None, '___sec50'),
+ ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+ ('Gradient Boosting, Examples', 2, None, '___sec52'),
+ ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
end of tocinfo -->
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
Final regressor code
Pros and cons of trees, pros
Disadvantages
- From a Single Tree to Many Trees, Meet the Jungle of Methods
+ Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
An Overview of Ensemble Methods
Bagging
More bagging
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
Using the Voting Classifier
Please, not the moons again! Voting and Bagging
Now Bagging
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Feature Importance
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
- AdaBoost Examples
- Gradient boosting: Basics
- Gradient Boosting, algorithm
- Gradient Boosting, Examples
- Gradient Boots with Early Stopping
- XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Making our own Bagging with Bootstrap
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
+ Figure to Illustrate the Iterative Classification Process
+ AdaBoost Examples
+ Gradient boosting: Basics
+ Gradient Boosting, algorithm
+ Gradient Boosting, Examples
+ Gradient Boots with Early Stopping
+ XGBoost: Extreme Gradient Boosting
+ Xgboost on the Cancer Data
@@ -234,24 +240,19 @@ MathJax.Hub.Config({
-Adaptive boosting: AdaBoost, Basic Algorithm
+Boosting, a Bird'e Eye
-The algorithm here is rather straightforward. Assume that our weak
-classifier is a decision tree and we consider a binary set of outputs
-with \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
-observations. Our design matrix is given in terms of the
-feature/predictor vectors
-\( \boldsymbol{X}=[\boldsymbol{x}_0\boldsymbol{x}_1\dots\boldsymbol{x}_{p-1} \). Finally, we define also a
-classifier determined by our data via a function \( G(\boldsymbol{X}) \). This function tells us how well we are able to classify our outputs/targets \( \boldsymbol{y} \).
+The basic idea is to combine weak classifiers in order to create a good
+classifier. With a weak classifier we often intend a classifier which
+produces results which are only slightly better than we would get by
+random guesses.
-We can then define the misclassification error \( \mathrm{err} \) as
-$$
-\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(\boldsymbol{X}_{i*}),
-$$
-
-where the function \( I() \) is one if we misclassify and zero if we classify correctly.
+This is done by applying in an iterative way a weak (or a standard
+classifier like decision trees) to modify the data. In each iteration
+we emphasize those observations which are misclassified by weighting
+them with a factor.
@@ -278,6 +279,8 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
54
55
56
+ ...
+ 57
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs047.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs047.html
index 35680be78..6cddc74f8 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs047.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs047.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
('Final regressor code', 2, None, '___sec28'),
('Pros and cons of trees, pros', 2, None, '___sec29'),
('Disadvantages', 2, None, '___sec30'),
- ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
2,
None,
'___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
None,
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
- ('Random forests', 2, None, '___sec39'),
- ('Random Forest Algorithm', 2, None, '___sec40'),
+ ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+ ('Random forests', 2, None, '___sec40'),
+ ('Random Forest Algorithm', 2, None, '___sec41'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec41'),
+ '___sec42'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec42'),
- ('Feature Importance', 2, None, '___sec43'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+ '___sec43'),
+ ('Bootstrap with Random Forests Instead of a Single Tree',
+ 2,
+ None,
+ '___sec44'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec45'),
- ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+ '___sec46'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec47'),
('Figure to Illustrate the Iterative Classification Process',
2,
None,
- '___sec47'),
- ('AdaBoost Examples', 2, None, '___sec48'),
- ('Gradient boosting: Basics', 2, None, '___sec49'),
- ('Gradient Boosting, algorithm', 2, None, '___sec50'),
- ('Gradient Boosting, Examples', 2, None, '___sec51'),
- ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
- ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
- ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+ '___sec48'),
+ ('AdaBoost Examples', 2, None, '___sec49'),
+ ('Gradient boosting: Basics', 2, None, '___sec50'),
+ ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+ ('Gradient Boosting, Examples', 2, None, '___sec52'),
+ ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
end of tocinfo -->
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
Final regressor code
Pros and cons of trees, pros
Disadvantages
- From a Single Tree to Many Trees, Meet the Jungle of Methods
+ Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
An Overview of Ensemble Methods
Bagging
More bagging
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
Using the Voting Classifier
Please, not the moons again! Voting and Bagging
Now Bagging
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Feature Importance
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
- AdaBoost Examples
- Gradient boosting: Basics
- Gradient Boosting, algorithm
- Gradient Boosting, Examples
- Gradient Boots with Early Stopping
- XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Making our own Bagging with Bootstrap
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
+ Figure to Illustrate the Iterative Classification Process
+ AdaBoost Examples
+ Gradient boosting: Basics
+ Gradient Boosting, algorithm
+ Gradient Boosting, Examples
+ Gradient Boots with Early Stopping
+ XGBoost: Extreme Gradient Boosting
+ Xgboost on the Cancer Data
@@ -234,42 +240,24 @@ MathJax.Hub.Config({
-Basic Steps of AdaBoost
+Adaptive boosting: AdaBoost, Basic Algorithm
-With the above definitions we are now ready to set up the algorithm for AdaBoost.
-The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases.
-
-
-- We start by initializing all weights to \( w_i = 1/n \), with \( i=0,1,2,\dots n-1 \). It is to see then that \( \sum_{i=0}^{n-1}w_i = 1 \).
-- We rewrite the misclassification error as
-
+The algorithm here is rather straightforward. Assume that our weak
+classifier is a decision tree and we consider a binary set of outputs
+with \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
+observations. Our design matrix is given in terms of the
+feature/predictor vectors
+\( \boldsymbol{X}=[\boldsymbol{x}_0\boldsymbol{x}_1\dots\boldsymbol{x}_{p-1} \). Finally, we define also a
+classifier determined by our data via a function \( G(\boldsymbol{X}) \). This function tells us how well we are able to classify our outputs/targets \( \boldsymbol{y} \).
+
+We can then define the misclassification error \( \mathrm{err} \) as
$$
-\mathrm{err}=\frac{\sum_{i=0}^{n-1}w_iI(y_i\ne G(\boldsymbol{X}_{i*})}{\sum_{i=0}^{n-1}w_i},
+\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(\boldsymbol{X}_{i*}),
$$
-
-
-- Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree.
-
-
- - Fit thus a given classifier to the training using the weights \( w_i \).
- - Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.
- - Define a quantity $\alpha_{m} = \log{(1-\mathrm{err})/\mathrm{err}}
- - Set the new weights to $w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(\boldsymbol{X}_{i*})}.
-
-
- Compute the new classifier $G(\boldsymbol{X})= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(\boldsymbol{X}_{i*}).
-
-
-For the iterations with \( m \le 2 \) the weights are modified
-individually at each steps. The obersvations which were misclassified
-at iteration \( m-1 \) have a weight which is larger than those which were
-classified properly. As this proceeds, the observations which were
-difficult to classifiy correctly are given a larger influence. Each
-new classificatio step \( m \) is then forced to concentrate on those
-observations that are missed in the previous iterations.
+where the function \( I() \) is one if we misclassify and zero if we classify correctly.
@@ -295,6 +283,7 @@ observations that are missed in the previous iterations.
54
55
56
+ 57
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs048.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs048.html
index 32e455b1e..e13cc85c8 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs048.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs048.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
('Final regressor code', 2, None, '___sec28'),
('Pros and cons of trees, pros', 2, None, '___sec29'),
('Disadvantages', 2, None, '___sec30'),
- ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
2,
None,
'___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
None,
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
- ('Random forests', 2, None, '___sec39'),
- ('Random Forest Algorithm', 2, None, '___sec40'),
+ ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+ ('Random forests', 2, None, '___sec40'),
+ ('Random Forest Algorithm', 2, None, '___sec41'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec41'),
+ '___sec42'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec42'),
- ('Feature Importance', 2, None, '___sec43'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+ '___sec43'),
+ ('Bootstrap with Random Forests Instead of a Single Tree',
+ 2,
+ None,
+ '___sec44'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec45'),
- ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+ '___sec46'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec47'),
('Figure to Illustrate the Iterative Classification Process',
2,
None,
- '___sec47'),
- ('AdaBoost Examples', 2, None, '___sec48'),
- ('Gradient boosting: Basics', 2, None, '___sec49'),
- ('Gradient Boosting, algorithm', 2, None, '___sec50'),
- ('Gradient Boosting, Examples', 2, None, '___sec51'),
- ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
- ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
- ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+ '___sec48'),
+ ('AdaBoost Examples', 2, None, '___sec49'),
+ ('Gradient boosting: Basics', 2, None, '___sec50'),
+ ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+ ('Gradient Boosting, Examples', 2, None, '___sec52'),
+ ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
end of tocinfo -->
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
Final regressor code
Pros and cons of trees, pros
Disadvantages
- From a Single Tree to Many Trees, Meet the Jungle of Methods
+ Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
An Overview of Ensemble Methods
Bagging
More bagging
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
Using the Voting Classifier
Please, not the moons again! Voting and Bagging
Now Bagging
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Feature Importance
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
- AdaBoost Examples
- Gradient boosting: Basics
- Gradient Boosting, algorithm
- Gradient Boosting, Examples
- Gradient Boots with Early Stopping
- XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Making our own Bagging with Bootstrap
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
+ Figure to Illustrate the Iterative Classification Process
+ AdaBoost Examples
+ Gradient boosting: Basics
+ Gradient Boosting, algorithm
+ Gradient Boosting, Examples
+ Gradient Boots with Early Stopping
+ XGBoost: Extreme Gradient Boosting
+ Xgboost on the Cancer Data
@@ -234,7 +240,42 @@ MathJax.Hub.Config({
-Figure to Illustrate the Iterative Classification Process
+Basic Steps of AdaBoost
+
+
+With the above definitions we are now ready to set up the algorithm for AdaBoost.
+The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases.
+
+
+- We start by initializing all weights to \( w_i = 1/n \), with \( i=0,1,2,\dots n-1 \). It is to see then that \( \sum_{i=0}^{n-1}w_i = 1 \).
+- We rewrite the misclassification error as
+
+
+$$
+\mathrm{err}=\frac{\sum_{i=0}^{n-1}w_iI(y_i\ne G(\boldsymbol{X}_{i*})}{\sum_{i=0}^{n-1}w_i},
+$$
+
+
+
+- Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree.
+
+
+ - Fit thus a given classifier to the training using the weights \( w_i \).
+ - Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.
+ - Define a quantity $\alpha_{m} = \log{(1-\mathrm{err})/\mathrm{err}}
+ - Set the new weights to $w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(\boldsymbol{X}_{i*})}.
+
+
+ Compute the new classifier $G(\boldsymbol{X})= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(\boldsymbol{X}_{i*}).
+
+
+For the iterations with \( m \le 2 \) the weights are modified
+individually at each steps. The obersvations which were misclassified
+at iteration \( m-1 \) have a weight which is larger than those which were
+classified properly. As this proceeds, the observations which were
+difficult to classifiy correctly are given a larger influence. Each
+new classification step \( m \) is then forced to concentrate on those
+observations that are missed in the previous iterations.
@@ -259,6 +300,7 @@ MathJax.Hub.Config({
54
55
56
+ 57
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs049.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs049.html
index f0981082a..4a6ad8a24 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs049.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs049.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
('Final regressor code', 2, None, '___sec28'),
('Pros and cons of trees, pros', 2, None, '___sec29'),
('Disadvantages', 2, None, '___sec30'),
- ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
2,
None,
'___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
None,
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
- ('Random forests', 2, None, '___sec39'),
- ('Random Forest Algorithm', 2, None, '___sec40'),
+ ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+ ('Random forests', 2, None, '___sec40'),
+ ('Random Forest Algorithm', 2, None, '___sec41'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec41'),
+ '___sec42'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec42'),
- ('Feature Importance', 2, None, '___sec43'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+ '___sec43'),
+ ('Bootstrap with Random Forests Instead of a Single Tree',
+ 2,
+ None,
+ '___sec44'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec45'),
- ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+ '___sec46'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec47'),
('Figure to Illustrate the Iterative Classification Process',
2,
None,
- '___sec47'),
- ('AdaBoost Examples', 2, None, '___sec48'),
- ('Gradient boosting: Basics', 2, None, '___sec49'),
- ('Gradient Boosting, algorithm', 2, None, '___sec50'),
- ('Gradient Boosting, Examples', 2, None, '___sec51'),
- ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
- ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
- ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+ '___sec48'),
+ ('AdaBoost Examples', 2, None, '___sec49'),
+ ('Gradient boosting: Basics', 2, None, '___sec50'),
+ ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+ ('Gradient Boosting, Examples', 2, None, '___sec52'),
+ ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
end of tocinfo -->
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
Final regressor code
Pros and cons of trees, pros
Disadvantages
- From a Single Tree to Many Trees, Meet the Jungle of Methods
+ Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
An Overview of Ensemble Methods
Bagging
More bagging
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
Using the Voting Classifier
Please, not the moons again! Voting and Bagging
Now Bagging
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Feature Importance
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
- AdaBoost Examples
- Gradient boosting: Basics
- Gradient Boosting, algorithm
- Gradient Boosting, Examples
- Gradient Boots with Early Stopping
- XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Making our own Bagging with Bootstrap
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
+ Figure to Illustrate the Iterative Classification Process
+ AdaBoost Examples
+ Gradient boosting: Basics
+ Gradient Boosting, algorithm
+ Gradient Boosting, Examples
+ Gradient Boots with Early Stopping
+ XGBoost: Extreme Gradient Boosting
+ Xgboost on the Cancer Data
@@ -234,46 +240,8 @@ MathJax.Hub.Config({
-AdaBoost Examples
+Figure to Illustrate the Iterative Classification Process
-
-Using Scikit-Learn it is easy to appply the adaptive boosting algorithm, as done here.
-
-
-
-
-
from sklearn.ensemble import AdaBoostClassifier
-
-ada_clf = AdaBoostClassifier(
- DecisionTreeClassifier(max_depth=1), n_estimators=200,
- algorithm="SAMME.R", learning_rate=0.5, random_state=42)
-ada_clf.fit(X_train, y_train)
-
-plot_decision_boundary(ada_clf, X, y)
-
-m = len(X_train)
-
-plt.figure(figsize=(11, 4))
-for subplot, learning_rate in ((121, 1), (122, 0.5)):
- sample_weights = np.ones(m)
- plt.subplot(subplot)
- for i in range(5):
- svm_clf = SVC(kernel="rbf", C=0.05, gamma="auto", random_state=42)
- svm_clf.fit(X_train, y_train, sample_weight=sample_weights)
- y_pred = svm_clf.predict(X_train)
- sample_weights[y_pred != y_train] *= (1 + learning_rate)
- plot_decision_boundary(svm_clf, X, y, alpha=0.2)
- plt.title("learning_rate = {}".format(learning_rate), fontsize=16)
- if subplot == 121:
- plt.text(-0.7, -0.65, "1", fontsize=14)
- plt.text(-0.6, -0.10, "2", fontsize=14)
- plt.text(-0.5, 0.10, "3", fontsize=14)
- plt.text(-0.4, 0.55, "4", fontsize=14)
- plt.text(-0.3, 0.90, "5", fontsize=14)
-
-save_fig("boosting_plot")
-plt.show()
-
@@ -296,6 +264,7 @@ plt.show()
54
55
56
+ 57
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs050.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs050.html
index 987ba0b72..4d7d6c3fa 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs050.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs050.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
('Final regressor code', 2, None, '___sec28'),
('Pros and cons of trees, pros', 2, None, '___sec29'),
('Disadvantages', 2, None, '___sec30'),
- ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
2,
None,
'___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
None,
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
- ('Random forests', 2, None, '___sec39'),
- ('Random Forest Algorithm', 2, None, '___sec40'),
+ ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+ ('Random forests', 2, None, '___sec40'),
+ ('Random Forest Algorithm', 2, None, '___sec41'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec41'),
+ '___sec42'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec42'),
- ('Feature Importance', 2, None, '___sec43'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+ '___sec43'),
+ ('Bootstrap with Random Forests Instead of a Single Tree',
+ 2,
+ None,
+ '___sec44'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec45'),
- ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+ '___sec46'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec47'),
('Figure to Illustrate the Iterative Classification Process',
2,
None,
- '___sec47'),
- ('AdaBoost Examples', 2, None, '___sec48'),
- ('Gradient boosting: Basics', 2, None, '___sec49'),
- ('Gradient Boosting, algorithm', 2, None, '___sec50'),
- ('Gradient Boosting, Examples', 2, None, '___sec51'),
- ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
- ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
- ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+ '___sec48'),
+ ('AdaBoost Examples', 2, None, '___sec49'),
+ ('Gradient boosting: Basics', 2, None, '___sec50'),
+ ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+ ('Gradient Boosting, Examples', 2, None, '___sec52'),
+ ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
end of tocinfo -->
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
Final regressor code
Pros and cons of trees, pros
Disadvantages
- From a Single Tree to Many Trees, Meet the Jungle of Methods
+ Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
An Overview of Ensemble Methods
Bagging
More bagging
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
Using the Voting Classifier
Please, not the moons again! Voting and Bagging
Now Bagging
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Feature Importance
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
- AdaBoost Examples
- Gradient boosting: Basics
- Gradient Boosting, algorithm
- Gradient Boosting, Examples
- Gradient Boots with Early Stopping
- XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Making our own Bagging with Bootstrap
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
+ Figure to Illustrate the Iterative Classification Process
+ AdaBoost Examples
+ Gradient boosting: Basics
+ Gradient Boosting, algorithm
+ Gradient Boosting, Examples
+ Gradient Boots with Early Stopping
+ XGBoost: Extreme Gradient Boosting
+ Xgboost on the Cancer Data
@@ -234,18 +240,46 @@ MathJax.Hub.Config({
-Gradient boosting: Basics
+AdaBoost Examples
-Gradient boosting is again a similar technique to Adapative boosting,
-it combines so-called weak classifiers or regressors into a strong
-method via a series of iterations.
+Using Scikit-Learn it is easy to appply the adaptive boosting algorithm, as done here.
-In order to understand the method, let us illustrate its basics by
-bringing back the essential steps in linear regression, where our cost
-function was the least squares function.
+
+
from sklearn.ensemble import AdaBoostClassifier
+
+ada_clf = AdaBoostClassifier(
+ DecisionTreeClassifier(max_depth=1), n_estimators=200,
+ algorithm="SAMME.R", learning_rate=0.5, random_state=42)
+ada_clf.fit(X_train, y_train)
+
+plot_decision_boundary(ada_clf, X, y)
+
+m = len(X_train)
+
+plt.figure(figsize=(11, 4))
+for subplot, learning_rate in ((121, 1), (122, 0.5)):
+ sample_weights = np.ones(m)
+ plt.subplot(subplot)
+ for i in range(5):
+ svm_clf = SVC(kernel="rbf", C=0.05, gamma="auto", random_state=42)
+ svm_clf.fit(X_train, y_train, sample_weight=sample_weights)
+ y_pred = svm_clf.predict(X_train)
+ sample_weights[y_pred != y_train] *= (1 + learning_rate)
+ plot_decision_boundary(svm_clf, X, y, alpha=0.2)
+ plt.title("learning_rate = {}".format(learning_rate), fontsize=16)
+ if subplot == 121:
+ plt.text(-0.7, -0.65, "1", fontsize=14)
+ plt.text(-0.6, -0.10, "2", fontsize=14)
+ plt.text(-0.5, 0.10, "3", fontsize=14)
+ plt.text(-0.4, 0.55, "4", fontsize=14)
+ plt.text(-0.3, 0.90, "5", fontsize=14)
+
+save_fig("boosting_plot")
+plt.show()
+
@@ -267,6 +301,7 @@ function was the least squares function.
54
55
56
+ 57
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs051.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs051.html
index 94873b2b4..fe4dd40f5 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs051.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs051.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
('Final regressor code', 2, None, '___sec28'),
('Pros and cons of trees, pros', 2, None, '___sec29'),
('Disadvantages', 2, None, '___sec30'),
- ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
2,
None,
'___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
None,
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
- ('Random forests', 2, None, '___sec39'),
- ('Random Forest Algorithm', 2, None, '___sec40'),
+ ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+ ('Random forests', 2, None, '___sec40'),
+ ('Random Forest Algorithm', 2, None, '___sec41'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec41'),
+ '___sec42'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec42'),
- ('Feature Importance', 2, None, '___sec43'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+ '___sec43'),
+ ('Bootstrap with Random Forests Instead of a Single Tree',
+ 2,
+ None,
+ '___sec44'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec45'),
- ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+ '___sec46'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec47'),
('Figure to Illustrate the Iterative Classification Process',
2,
None,
- '___sec47'),
- ('AdaBoost Examples', 2, None, '___sec48'),
- ('Gradient boosting: Basics', 2, None, '___sec49'),
- ('Gradient Boosting, algorithm', 2, None, '___sec50'),
- ('Gradient Boosting, Examples', 2, None, '___sec51'),
- ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
- ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
- ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+ '___sec48'),
+ ('AdaBoost Examples', 2, None, '___sec49'),
+ ('Gradient boosting: Basics', 2, None, '___sec50'),
+ ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+ ('Gradient Boosting, Examples', 2, None, '___sec52'),
+ ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
end of tocinfo -->
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
Final regressor code
Pros and cons of trees, pros
Disadvantages
- From a Single Tree to Many Trees, Meet the Jungle of Methods
+ Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
An Overview of Ensemble Methods
Bagging
More bagging
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
Using the Voting Classifier
Please, not the moons again! Voting and Bagging
Now Bagging
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Feature Importance
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
- AdaBoost Examples
- Gradient boosting: Basics
- Gradient Boosting, algorithm
- Gradient Boosting, Examples
- Gradient Boots with Early Stopping
- XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Making our own Bagging with Bootstrap
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
+ Figure to Illustrate the Iterative Classification Process
+ AdaBoost Examples
+ Gradient boosting: Basics
+ Gradient Boosting, algorithm
+ Gradient Boosting, Examples
+ Gradient Boots with Early Stopping
+ XGBoost: Extreme Gradient Boosting
+ Xgboost on the Cancer Data
@@ -234,30 +240,19 @@ MathJax.Hub.Config({
-Gradient Boosting, algorithm
+Gradient boosting: Basics
-Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard least squares function
-$$
-C(\boldsymbol{y},\boldsymbol{f})=\frac{1}{n}\sum_{i=0}^{n-1}(y_i-f(x_i))^2.
-$$
+Gradient boosting is again a similar technique to Adapative boosting,
+it combines so-called weak classifiers or regressors into a strong
+method via a series of iterations.
-The way we proceed in an iterative fashion is to
-
-
-- Initialize our estimate by \( f_0(x)=0 \).
-- For \( m=1:M \), we
-
-
- - compute the negative gradient vector \( \boldsymbol{u}_m = -\partial C(\boldsymbol{y},\boldsymbol{f})/\partial \boldsymbol{f}(x) \) at $f(x) = f_{m-1}(x);
- - fit the so-called base-learner to the negative gradient \( h_m(u_m,x) \);
- - update the estimate \( f_m(x) = f_{m-1}(x)+\nu h_m(u_m,x) \);
-
-
- The final estimate is then \( f_M(x) = \sum_{m=1}^M\nu h_m(u_m,x) \).
-
+In order to understand the method, let us illustrate its basics by
+bringing back the essential steps in linear regression, where our cost
+function was the least squares function.
+
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs052.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs052.html
index a1d6b07ec..0f3729124 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs052.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs052.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
('Final regressor code', 2, None, '___sec28'),
('Pros and cons of trees, pros', 2, None, '___sec29'),
('Disadvantages', 2, None, '___sec30'),
- ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
2,
None,
'___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
None,
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
- ('Random forests', 2, None, '___sec39'),
- ('Random Forest Algorithm', 2, None, '___sec40'),
+ ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+ ('Random forests', 2, None, '___sec40'),
+ ('Random Forest Algorithm', 2, None, '___sec41'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec41'),
+ '___sec42'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec42'),
- ('Feature Importance', 2, None, '___sec43'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+ '___sec43'),
+ ('Bootstrap with Random Forests Instead of a Single Tree',
+ 2,
+ None,
+ '___sec44'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec45'),
- ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+ '___sec46'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec47'),
('Figure to Illustrate the Iterative Classification Process',
2,
None,
- '___sec47'),
- ('AdaBoost Examples', 2, None, '___sec48'),
- ('Gradient boosting: Basics', 2, None, '___sec49'),
- ('Gradient Boosting, algorithm', 2, None, '___sec50'),
- ('Gradient Boosting, Examples', 2, None, '___sec51'),
- ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
- ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
- ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+ '___sec48'),
+ ('AdaBoost Examples', 2, None, '___sec49'),
+ ('Gradient boosting: Basics', 2, None, '___sec50'),
+ ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+ ('Gradient Boosting, Examples', 2, None, '___sec52'),
+ ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
end of tocinfo -->
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
Final regressor code
Pros and cons of trees, pros
Disadvantages
- From a Single Tree to Many Trees, Meet the Jungle of Methods
+ Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
An Overview of Ensemble Methods
Bagging
More bagging
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
Using the Voting Classifier
Please, not the moons again! Voting and Bagging
Now Bagging
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Feature Importance
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
- AdaBoost Examples
- Gradient boosting: Basics
- Gradient Boosting, algorithm
- Gradient Boosting, Examples
- Gradient Boots with Early Stopping
- XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Making our own Bagging with Bootstrap
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
+ Figure to Illustrate the Iterative Classification Process
+ AdaBoost Examples
+ Gradient boosting: Basics
+ Gradient Boosting, algorithm
+ Gradient Boosting, Examples
+ Gradient Boots with Early Stopping
+ XGBoost: Extreme Gradient Boosting
+ Xgboost on the Cancer Data
@@ -234,94 +240,30 @@ MathJax.Hub.Config({
-Gradient Boosting, Examples
+Gradient Boosting, algorithm
+
+Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard least squares function
+$$
+C(\boldsymbol{y},\boldsymbol{f})=\frac{1}{n}\sum_{i=0}^{n-1}(y_i-f(x_i))^2.
+$$
-
-
np.random.seed(42)
-X = np.random.rand(100, 1) - 0.5
-y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)
-
-from sklearn.tree import DecisionTreeRegressor
-
-tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg1.fit(X, y)
-
-y2 = y - tree_reg1.predict(X)
-tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg2.fit(X, y2)
-
-y3 = y2 - tree_reg2.predict(X)
-tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg3.fit(X, y3)
-
-X_new = np.array([[0.8]])
-y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))
-
-def plot_predictions(regressors, X, y, axes, label=None, style="r-", data_style="b.", data_label=None):
- x1 = np.linspace(axes[0], axes[1], 500)
- y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)
- plt.plot(X[:, 0], y, data_style, label=data_label)
- plt.plot(x1, y_pred, style, linewidth=2, label=label)
- if label or data_label:
- plt.legend(loc="upper center", fontsize=16)
- plt.axis(axes)
-
-plt.figure(figsize=(11,11))
-
-plt.subplot(321)
-plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h_1(x_1)$", style="g-", data_label="Training set")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-plt.title("Residuals and tree predictions", fontsize=16)
-
-plt.subplot(322)
-plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1)$", data_label="Training set")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-plt.title("Ensemble predictions", fontsize=16)
-
-plt.subplot(323)
-plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_2(x_1)$", style="g-", data_style="k+", data_label="Residuals")
-plt.ylabel("$y - h_1(x_1)$", fontsize=16)
-
-plt.subplot(324)
-plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1)$")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-
-plt.subplot(325)
-plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_3(x_1)$", style="g-", data_style="k+")
-plt.ylabel("$y - h_1(x_1) - h_2(x_1)$", fontsize=16)
-plt.xlabel("$x_1$", fontsize=16)
-
-plt.subplot(326)
-plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$")
-plt.xlabel("$x_1$", fontsize=16)
-plt.ylabel("$y$", fontsize=16, rotation=0)
-
-save_fig("gradient_boosting_plot")
-plt.show()
-
-from sklearn.ensemble import GradientBoostingRegressor
-
-gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0, random_state=42)
-gbrt.fit(X, y)
-
-gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)
-gbrt_slow.fit(X, y)
-
-plt.figure(figsize=(11,4))
-
-plt.subplot(121)
-plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="Ensemble predictions")
-plt.title("learning_rate={}, n_estimators={}".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)
-
-plt.subplot(122)
-plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
-plt.title("learning_rate={}, n_estimators={}".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)
-
-save_fig("gbrt_learning_rate_plot")
-plt.show()
-
+The way we proceed in an iterative fashion is to
+
+
+- Initialize our estimate by \( f_0(x)=0 \).
+- For \( m=1:M \), we
+
+
+ - compute the negative gradient vector \( \boldsymbol{u}_m = -\partial C(\boldsymbol{y},\boldsymbol{f})/\partial \boldsymbol{f}(x) \) at $f(x) = f_{m-1}(x);
+ - fit the so-called base-learner to the negative gradient \( h_m(u_m,x) \);
+ - update the estimate \( f_m(x) = f_{m-1}(x)+\nu h_m(u_m,x) \);
+
+
+ The final estimate is then \( f_M(x) = \sum_{m=1}^M\nu h_m(u_m,x) \).
+
+
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs053.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs053.html
index b0c98925a..591f2cd60 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs053.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs053.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
('Final regressor code', 2, None, '___sec28'),
('Pros and cons of trees, pros', 2, None, '___sec29'),
('Disadvantages', 2, None, '___sec30'),
- ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
2,
None,
'___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
None,
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
- ('Random forests', 2, None, '___sec39'),
- ('Random Forest Algorithm', 2, None, '___sec40'),
+ ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+ ('Random forests', 2, None, '___sec40'),
+ ('Random Forest Algorithm', 2, None, '___sec41'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec41'),
+ '___sec42'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec42'),
- ('Feature Importance', 2, None, '___sec43'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+ '___sec43'),
+ ('Bootstrap with Random Forests Instead of a Single Tree',
+ 2,
+ None,
+ '___sec44'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec45'),
- ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+ '___sec46'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec47'),
('Figure to Illustrate the Iterative Classification Process',
2,
None,
- '___sec47'),
- ('AdaBoost Examples', 2, None, '___sec48'),
- ('Gradient boosting: Basics', 2, None, '___sec49'),
- ('Gradient Boosting, algorithm', 2, None, '___sec50'),
- ('Gradient Boosting, Examples', 2, None, '___sec51'),
- ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
- ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
- ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+ '___sec48'),
+ ('AdaBoost Examples', 2, None, '___sec49'),
+ ('Gradient boosting: Basics', 2, None, '___sec50'),
+ ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+ ('Gradient Boosting, Examples', 2, None, '___sec52'),
+ ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
end of tocinfo -->
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
Final regressor code
Pros and cons of trees, pros
Disadvantages
- From a Single Tree to Many Trees, Meet the Jungle of Methods
+ Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
An Overview of Ensemble Methods
Bagging
More bagging
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
Using the Voting Classifier
Please, not the moons again! Voting and Bagging
Now Bagging
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Feature Importance
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
- AdaBoost Examples
- Gradient boosting: Basics
- Gradient Boosting, algorithm
- Gradient Boosting, Examples
- Gradient Boots with Early Stopping
- XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Making our own Bagging with Bootstrap
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
+ Figure to Illustrate the Iterative Classification Process
+ AdaBoost Examples
+ Gradient boosting: Basics
+ Gradient Boosting, algorithm
+ Gradient Boosting, Examples
+ Gradient Boots with Early Stopping
+ XGBoost: Extreme Gradient Boosting
+ Xgboost on the Cancer Data
@@ -234,66 +240,92 @@ MathJax.Hub.Config({
-Gradient Boots with Early Stopping
+Gradient Boosting, Examples
-
from sklearn.model_selection import train_test_split
-from sklearn.metrics import mean_squared_error
+np.random.seed(42)
+X = np.random.rand(100, 1) - 0.5
+y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)
-X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=49)
+from sklearn.tree import DecisionTreeRegressor
-gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=120, random_state=42)
-gbrt.fit(X_train, y_train)
+tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg1.fit(X, y)
-errors = [mean_squared_error(y_val, y_pred)
- for y_pred in gbrt.staged_predict(X_val)]
-bst_n_estimators = np.argmin(errors) + 1
+y2 = y - tree_reg1.predict(X)
+tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg2.fit(X, y2)
-gbrt_best = GradientBoostingRegressor(max_depth=2,n_estimators=bst_n_estimators, random_state=42)
-gbrt_best.fit(X_train, y_train)
+y3 = y2 - tree_reg2.predict(X)
+tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg3.fit(X, y3)
-min_error = np.min(errors)
-plt.figure(figsize=(11, 4))
+X_new = np.array([[0.8]])
+y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))
-plt.subplot(121)
-plt.plot(errors, "b.-")
-plt.plot([bst_n_estimators, bst_n_estimators], [0, min_error], "k--")
-plt.plot([0, 120], [min_error, min_error], "k--")
-plt.plot(bst_n_estimators, min_error, "ko")
-plt.text(bst_n_estimators, min_error*1.2, "Minimum", ha="center", fontsize=14)
-plt.axis([0, 120, 0, 0.01])
-plt.xlabel("Number of trees")
-plt.title("Validation error", fontsize=14)
+def plot_predictions(regressors, X, y, axes, label=None, style="r-", data_style="b.", data_label=None):
+ x1 = np.linspace(axes[0], axes[1], 500)
+ y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)
+ plt.plot(X[:, 0], y, data_style, label=data_label)
+ plt.plot(x1, y_pred, style, linewidth=2, label=label)
+ if label or data_label:
+ plt.legend(loc="upper center", fontsize=16)
+ plt.axis(axes)
-plt.subplot(122)
-plot_predictions([gbrt_best], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
-plt.title("Best model (%d trees)" % bst_n_estimators, fontsize=14)
+plt.figure(figsize=(11,11))
-save_fig("early_stopping_gbrt_plot")
+plt.subplot(321)
+plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h_1(x_1)$", style="g-", data_label="Training set")
+plt.ylabel("$y$", fontsize=16, rotation=0)
+plt.title("Residuals and tree predictions", fontsize=16)
+
+plt.subplot(322)
+plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1)$", data_label="Training set")
+plt.ylabel("$y$", fontsize=16, rotation=0)
+plt.title("Ensemble predictions", fontsize=16)
+
+plt.subplot(323)
+plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_2(x_1)$", style="g-", data_style="k+", data_label="Residuals")
+plt.ylabel("$y - h_1(x_1)$", fontsize=16)
+
+plt.subplot(324)
+plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1)$")
+plt.ylabel("$y$", fontsize=16, rotation=0)
+
+plt.subplot(325)
+plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_3(x_1)$", style="g-", data_style="k+")
+plt.ylabel("$y - h_1(x_1) - h_2(x_1)$", fontsize=16)
+plt.xlabel("$x_1$", fontsize=16)
+
+plt.subplot(326)
+plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$")
+plt.xlabel("$x_1$", fontsize=16)
+plt.ylabel("$y$", fontsize=16, rotation=0)
+
+save_fig("gradient_boosting_plot")
plt.show()
+from sklearn.ensemble import GradientBoostingRegressor
-gbrt = GradientBoostingRegressor(max_depth=2, warm_start=True, random_state=42)
+gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0, random_state=42)
+gbrt.fit(X, y)
-min_val_error = float("inf")
-error_going_up = 0
-for n_estimators in range(1, 120):
- gbrt.n_estimators = n_estimators
- gbrt.fit(X_train, y_train)
- y_pred = gbrt.predict(X_val)
- val_error = mean_squared_error(y_val, y_pred)
- if val_error < min_val_error:
- min_val_error = val_error
- error_going_up = 0
- else:
- error_going_up += 1
- if error_going_up == 5:
- break # early stopping
+gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)
+gbrt_slow.fit(X, y)
+plt.figure(figsize=(11,4))
-print(gbrt.n_estimators)
-print("Minimum validation MSE:", min_val_error)
+plt.subplot(121)
+plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="Ensemble predictions")
+plt.title("learning_rate={}, n_estimators={}".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)
+
+plt.subplot(122)
+plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
+plt.title("learning_rate={}, n_estimators={}".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)
+
+save_fig("gbrt_learning_rate_plot")
+plt.show()
@@ -313,6 +345,7 @@ error_going_up = 54
55
56
+ 57
»
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-bs.html b/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
index cd7951257..a15bd51fb 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
@@ -85,7 +85,8 @@ Automatically generated HTML file from DocOnce source
('Final regressor code', 2, None, '___sec28'),
('Pros and cons of trees, pros', 2, None, '___sec29'),
('Disadvantages', 2, None, '___sec30'),
- ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
2,
None,
'___sec31'),
@@ -99,34 +100,38 @@ Automatically generated HTML file from DocOnce source
None,
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
- ('Random forests', 2, None, '___sec39'),
- ('Random Forest Algorithm', 2, None, '___sec40'),
+ ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+ ('Random forests', 2, None, '___sec40'),
+ ('Random Forest Algorithm', 2, None, '___sec41'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec41'),
+ '___sec42'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec42'),
- ('Feature Importance', 2, None, '___sec43'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+ '___sec43'),
+ ('Bootstrap with Random Forests Instead of a Single Tree',
+ 2,
+ None,
+ '___sec44'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec45'),
- ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+ '___sec46'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec47'),
('Figure to Illustrate the Iterative Classification Process',
2,
None,
- '___sec47'),
- ('AdaBoost Examples', 2, None, '___sec48'),
- ('Gradient boosting: Basics', 2, None, '___sec49'),
- ('Gradient Boosting, algorithm', 2, None, '___sec50'),
- ('Gradient Boosting, Examples', 2, None, '___sec51'),
- ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
- ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
- ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+ '___sec48'),
+ ('AdaBoost Examples', 2, None, '___sec49'),
+ ('Gradient boosting: Basics', 2, None, '___sec50'),
+ ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+ ('Gradient Boosting, Examples', 2, None, '___sec52'),
+ ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
end of tocinfo -->
@@ -195,7 +200,7 @@ MathJax.Hub.Config({
Final regressor code
Pros and cons of trees, pros
Disadvantages
- From a Single Tree to Many Trees, Meet the Jungle of Methods
+ Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
An Overview of Ensemble Methods
Bagging
More bagging
@@ -203,22 +208,23 @@ MathJax.Hub.Config({
Using the Voting Classifier
Please, not the moons again! Voting and Bagging
Now Bagging
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Feature Importance
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
- AdaBoost Examples
- Gradient boosting: Basics
- Gradient Boosting, algorithm
- Gradient Boosting, Examples
- Gradient Boots with Early Stopping
- XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Making our own Bagging with Bootstrap
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
+ Figure to Illustrate the Iterative Classification Process
+ AdaBoost Examples
+ Gradient boosting: Basics
+ Gradient Boosting, algorithm
+ Gradient Boosting, Examples
+ Gradient Boots with Early Stopping
+ XGBoost: Extreme Gradient Boosting
+ Xgboost on the Cancer Data
@@ -253,7 +259,7 @@ MathJax.Hub.Config({
[2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
-
Nov 6, 2019
Nov 7, 2019
@@ -277,7 +283,7 @@ MathJax.Hub.Config({
9
10
...
- 56
+ 57
»
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html b/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
index 00d97656f..496aefa10 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
@@ -148,7 +148,7 @@ MathJax.Hub.Config({
[2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
-
Nov 6, 2019
Nov 7, 2019
@@ -698,6 +698,8 @@ os.system(cmd)
Algorithms for Setting up Decision Trees
+
+
Two algorithms stand out in the set up of decision trees:
@@ -706,7 +708,7 @@ Two algorithms stand out in the set up of decision trees:
-We discuss both algorithms with applications here. The popular library -Scikit-Learn_ uses the CART algorithm. For classification problems you can use either the gini index or the entropy to split a tree in two branches.
+We discuss both algorithms with applications here. The popular library Scikit-Learn uses the CART algorithm. For classification problems you can use either the gini index or the entropy to split a tree in two branches.
@@ -1439,7 +1441,7 @@ However, by aggregating many decision trees, using methods like bagging, random
-From a Single Tree to Many Trees, Meet the Jungle of Methods
+Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
As stated above and seen in many of the examples discussed here about
@@ -1455,7 +1457,7 @@ machine learning algorithms or just use one of them to construct forests and jun
Voting classifiers
Bagging and Pasting
Random forests
- Boosting methods
+ Boosting methods, from adaptive to Extreme Gradient Boosting (XGBoost)
@@ -1714,7 +1716,68 @@ plt.show()
-Random forests
+Making our own Bagging with Bootstrap
+
+
+
+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
+
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = DecisionTreeRegressor(max_depth=5)
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
+
+
+Random forests
Random forests provide an improvement over bagged trees by way of a
@@ -1760,7 +1823,7 @@ this setting.
-Random Forest Algorithm
+Random Forest Algorithm
The algorithm described here can be applied to both classification and regression problems.
@@ -1791,7 +1854,7 @@ We will grow of forest of say \( M \) trees.
-Random Forests Compared with other Methods on the Cancer Data
+Random Forests Compared with other Methods on the Cancer Data
@@ -1854,7 +1917,6 @@ accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=import scikitplot as skplt
y_pred = Random_Forest_model.predict(X_test_scaled)
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
-#<matplotlib.axes._subplots.AxesSubplot object at 0x7fe967d64490>
plt.show()
y_probas = Random_Forest_model.predict_proba(X_test_scaled)
skplt.metrics.plot_roc(y_test, y_probas)
@@ -1866,7 +1928,7 @@ plt.show()
-Compare Bagging on Trees with Random Forests
+Compare Bagging on Trees with Random Forests
@@ -1889,15 +1951,68 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
-Feature Importance
+Bootstrap with Random Forests Instead of a Single Tree
-Example will be added here.
+
+
+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.ensemble import RandomForestRegressor
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = RandomForestRegressor()
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
-Boosting, a Bird'e Eye
+Boosting, a Bird'e Eye
The basic idea is to combine weak classifiers in order to create a good
@@ -1914,7 +2029,7 @@ them with a factor.
-Adaptive boosting: AdaBoost, Basic Algorithm
+Adaptive boosting: AdaBoost, Basic Algorithm
The algorithm here is rather straightforward. Assume that our weak
@@ -1938,7 +2053,7 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
-Basic Steps of AdaBoost
+Basic Steps of AdaBoost
With the above definitions we are now ready to set up the algorithm for AdaBoost.
@@ -1973,18 +2088,18 @@ individually at each steps. The obersvations which were misclassified
at iteration \( m-1 \) have a weight which is larger than those which were
classified properly. As this proceeds, the observations which were
difficult to classifiy correctly are given a larger influence. Each
-new classificatio step \( m \) is then forced to concentrate on those
+new classification step \( m \) is then forced to concentrate on those
observations that are missed in the previous iterations.
-Figure to Illustrate the Iterative Classification Process
+Figure to Illustrate the Iterative Classification Process
-AdaBoost Examples
+AdaBoost Examples
Using Scikit-Learn it is easy to appply the adaptive boosting algorithm, as done here.
@@ -2028,7 +2143,7 @@ plt.show()
-Gradient boosting: Basics
+Gradient boosting: Basics
Gradient boosting is again a similar technique to Adapative boosting,
@@ -2043,7 +2158,7 @@ function was the least squares function.
-Gradient Boosting, algorithm
+Gradient Boosting, algorithm
Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard least squares function
@@ -2071,7 +2186,7 @@ The way we proceed in an iterative fashion is to
-Gradient Boosting, Examples
+Gradient Boosting, Examples
@@ -2162,7 +2277,7 @@ plt.show()
-Gradient Boots with Early Stopping
+Gradient Boots with Early Stopping
@@ -2227,7 +2342,7 @@ error_going_up = 0
-XGBoost: Extreme Gradient Boosting
+XGBoost: Extreme Gradient Boosting
XGBoost or Extreme Gradient
@@ -2248,7 +2363,7 @@ It is now the algorithm which wins essentially all ML competitions!!!
-Xgboost on the Cancer Data
+Xgboost on the Cancer Data
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html b/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html
index 690ca8a77..b2c1e8ea1 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html
@@ -105,7 +105,8 @@ div { text-align: justify; text-justify: inter-word; }
('Final regressor code', 2, None, '___sec28'),
('Pros and cons of trees, pros', 2, None, '___sec29'),
('Disadvantages', 2, None, '___sec30'),
- ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
2,
None,
'___sec31'),
@@ -119,34 +120,38 @@ div { text-align: justify; text-justify: inter-word; }
None,
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
- ('Random forests', 2, None, '___sec39'),
- ('Random Forest Algorithm', 2, None, '___sec40'),
+ ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+ ('Random forests', 2, None, '___sec40'),
+ ('Random Forest Algorithm', 2, None, '___sec41'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec41'),
+ '___sec42'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec42'),
- ('Feature Importance', 2, None, '___sec43'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+ '___sec43'),
+ ('Bootstrap with Random Forests Instead of a Single Tree',
+ 2,
+ None,
+ '___sec44'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec45'),
- ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+ '___sec46'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec47'),
('Figure to Illustrate the Iterative Classification Process',
2,
None,
- '___sec47'),
- ('AdaBoost Examples', 2, None, '___sec48'),
- ('Gradient boosting: Basics', 2, None, '___sec49'),
- ('Gradient Boosting, algorithm', 2, None, '___sec50'),
- ('Gradient Boosting, Examples', 2, None, '___sec51'),
- ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
- ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
- ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+ '___sec48'),
+ ('AdaBoost Examples', 2, None, '___sec49'),
+ ('Gradient boosting: Basics', 2, None, '___sec50'),
+ ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+ ('Gradient Boosting, Examples', 2, None, '___sec52'),
+ ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
end of tocinfo -->
@@ -188,7 +193,7 @@ MathJax.Hub.Config({
[2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
-
Nov 6, 2019
Nov 7, 2019
@@ -711,6 +716,8 @@ os.system(cmd)
Algorithms for Setting up Decision Trees
+
+
Two algorithms stand out in the set up of decision trees:
@@ -718,7 +725,7 @@ Two algorithms stand out in the set up of decision trees:
- The ID3 algorithm based on the computation of the information gain for classification
-We discuss both algorithms with applications here. The popular library -Scikit-Learn_ uses the CART algorithm. For classification problems you can use either the gini index or the entropy to split a tree in two branches.
+We discuss both algorithms with applications here. The popular library Scikit-Learn uses the CART algorithm. For classification problems you can use either the gini index or the entropy to split a tree in two branches.
@@ -1439,7 +1446,7 @@ However, by aggregating many decision trees, using methods like bagging, random
-
From a Single Tree to Many Trees, Meet the Jungle of Methods
+Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
As stated above and seen in many of the examples discussed here about
@@ -1455,7 +1462,7 @@ machine learning algorithms or just use one of them to construct forests and jun
Voting classifiers
Bagging and Pasting
Random forests
- Boosting methods
+ Boosting methods, from adaptive to Extreme Gradient Boosting (XGBoost)
We discuss these methods here.
@@ -1709,7 +1716,67 @@ plt.show()
-
Random forests
+Making our own Bagging with Bootstrap
+
+
+
+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
+
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = DecisionTreeRegressor(max_depth=5)
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
+
+
Random forests
Random forests provide an improvement over bagged trees by way of a
@@ -1753,7 +1820,7 @@ this setting.
-
Random Forest Algorithm
+Random Forest Algorithm
The algorithm described here can be applied to both classification and regression problems.
@@ -1779,7 +1846,7 @@ We will grow of forest of say \( M \) trees.
-
Random Forests Compared with other Methods on the Cancer Data
+Random Forests Compared with other Methods on the Cancer Data
@@ -1842,7 +1909,6 @@ accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=import scikitplot as skplt
y_pred = Random_Forest_model.predict(X_test_scaled)
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
-#<matplotlib.axes._subplots.AxesSubplot object at 0x7fe967d64490>
plt.show()
y_probas = Random_Forest_model.predict_proba(X_test_scaled)
skplt.metrics.plot_roc(y_test, y_probas)
@@ -1853,7 +1919,7 @@ plt.show()
-
Compare Bagging on Trees with Random Forests
+Compare Bagging on Trees with Random Forests
@@ -1875,15 +1941,67 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
-
Feature Importance
+Bootstrap with Random Forests Instead of a Single Tree
-Example will be added here.
+
+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.ensemble import RandomForestRegressor
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = RandomForestRegressor()
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
-
Boosting, a Bird'e Eye
+Boosting, a Bird'e Eye
The basic idea is to combine weak classifiers in order to create a good
@@ -1900,7 +2018,7 @@ them with a factor.
-
Adaptive boosting: AdaBoost, Basic Algorithm
+Adaptive boosting: AdaBoost, Basic Algorithm
The algorithm here is rather straightforward. Assume that our weak
@@ -1922,7 +2040,7 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
-
Basic Steps of AdaBoost
+Basic Steps of AdaBoost
With the above definitions we are now ready to set up the algorithm for AdaBoost.
@@ -1956,18 +2074,18 @@ individually at each steps. The obersvations which were misclassified
at iteration \( m-1 \) have a weight which is larger than those which were
classified properly. As this proceeds, the observations which were
difficult to classifiy correctly are given a larger influence. Each
-new classificatio step \( m \) is then forced to concentrate on those
+new classification step \( m \) is then forced to concentrate on those
observations that are missed in the previous iterations.
-
Figure to Illustrate the Iterative Classification Process
+Figure to Illustrate the Iterative Classification Process
-
AdaBoost Examples
+AdaBoost Examples
Using Scikit-Learn it is easy to appply the adaptive boosting algorithm, as done here.
@@ -2010,7 +2128,7 @@ plt.show()
-
Gradient boosting: Basics
+Gradient boosting: Basics
Gradient boosting is again a similar technique to Adapative boosting,
@@ -2025,7 +2143,7 @@ function was the least squares function.
-
Gradient Boosting, algorithm
+Gradient Boosting, algorithm
Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard least squares function
@@ -2051,7 +2169,7 @@ The way we proceed in an iterative fashion is to
-
Gradient Boosting, Examples
+Gradient Boosting, Examples
@@ -2141,7 +2259,7 @@ plt.show()
-
Gradient Boots with Early Stopping
+Gradient Boots with Early Stopping
@@ -2205,7 +2323,7 @@ error_going_up = 0
-
XGBoost: Extreme Gradient Boosting
+XGBoost: Extreme Gradient Boosting
XGBoost or Extreme Gradient
@@ -2226,7 +2344,7 @@ It is now the algorithm which wins essentially all ML competitions!!!
-
Xgboost on the Cancer Data
+Xgboost on the Cancer Data
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees.html b/doc/pub/DecisionTrees/html/DecisionTrees.html
index 186eea8a1..ede4f121d 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees.html
@@ -110,7 +110,8 @@ div { text-align: justify; text-justify: inter-word; }
('Final regressor code', 2, None, '___sec28'),
('Pros and cons of trees, pros', 2, None, '___sec29'),
('Disadvantages', 2, None, '___sec30'),
- ('From a Single Tree to Many Trees, Meet the Jungle of Methods',
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
2,
None,
'___sec31'),
@@ -124,34 +125,38 @@ div { text-align: justify; text-justify: inter-word; }
None,
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
- ('Random forests', 2, None, '___sec39'),
- ('Random Forest Algorithm', 2, None, '___sec40'),
+ ('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
+ ('Random forests', 2, None, '___sec40'),
+ ('Random Forest Algorithm', 2, None, '___sec41'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec41'),
+ '___sec42'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec42'),
- ('Feature Importance', 2, None, '___sec43'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec44'),
+ '___sec43'),
+ ('Bootstrap with Random Forests Instead of a Single Tree',
+ 2,
+ None,
+ '___sec44'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec45'),
- ('Basic Steps of AdaBoost', 2, None, '___sec46'),
+ '___sec46'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec47'),
('Figure to Illustrate the Iterative Classification Process',
2,
None,
- '___sec47'),
- ('AdaBoost Examples', 2, None, '___sec48'),
- ('Gradient boosting: Basics', 2, None, '___sec49'),
- ('Gradient Boosting, algorithm', 2, None, '___sec50'),
- ('Gradient Boosting, Examples', 2, None, '___sec51'),
- ('Gradient Boots with Early Stopping', 2, None, '___sec52'),
- ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec53'),
- ('Xgboost on the Cancer Data', 2, None, '___sec54')]}
+ '___sec48'),
+ ('AdaBoost Examples', 2, None, '___sec49'),
+ ('Gradient boosting: Basics', 2, None, '___sec50'),
+ ('Gradient Boosting, algorithm', 2, None, '___sec51'),
+ ('Gradient Boosting, Examples', 2, None, '___sec52'),
+ ('Gradient Boots with Early Stopping', 2, None, '___sec53'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
end of tocinfo -->
@@ -193,7 +198,7 @@ MathJax.Hub.Config({
[2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
-
Nov 6, 2019
Nov 7, 2019
@@ -716,6 +721,8 @@ os.system(cmd)
Algorithms for Setting up Decision Trees
+
+
Two algorithms stand out in the set up of decision trees:
@@ -723,7 +730,7 @@ Two algorithms stand out in the set up of decision trees:
- The ID3 algorithm based on the computation of the information gain for classification
-We discuss both algorithms with applications here. The popular library -Scikit-Learn_ uses the CART algorithm. For classification problems you can use either the gini index or the entropy to split a tree in two branches.
+We discuss both algorithms with applications here. The popular library Scikit-Learn uses the CART algorithm. For classification problems you can use either the gini index or the entropy to split a tree in two branches.
@@ -1444,7 +1451,7 @@ However, by aggregating many decision trees, using methods like bagging, random
-
From a Single Tree to Many Trees, Meet the Jungle of Methods
+Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
As stated above and seen in many of the examples discussed here about
@@ -1460,7 +1467,7 @@ machine learning algorithms or just use one of them to construct forests and jun
Voting classifiers
Bagging and Pasting
Random forests
- Boosting methods
+ Boosting methods, from adaptive to Extreme Gradient Boosting (XGBoost)
We discuss these methods here.
@@ -1714,7 +1721,67 @@ plt.show()
-
Random forests
+Making our own Bagging with Bootstrap
+
+
+
+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
+
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = DecisionTreeRegressor(max_depth=5)
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
+
+
Random forests
Random forests provide an improvement over bagged trees by way of a
@@ -1758,7 +1825,7 @@ this setting.
-
Random Forest Algorithm
+Random Forest Algorithm
The algorithm described here can be applied to both classification and regression problems.
@@ -1784,7 +1851,7 @@ We will grow of forest of say \( M \) trees.
-
Random Forests Compared with other Methods on the Cancer Data
+Random Forests Compared with other Methods on the Cancer Data
@@ -1847,7 +1914,6 @@ accuracy = cross_validate(Random_Forest_mode
import scikitplot as skplt
y_pred = Random_Forest_model.predict(X_test_scaled)
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
-#<matplotlib.axes._subplots.AxesSubplot object at 0x7fe967d64490>
plt.show()
y_probas = Random_Forest_model.predict_proba(X_test_scaled)
skplt.metrics.plot_roc(y_test, y_probas)
@@ -1858,7 +1924,7 @@ plt.show()
-
Compare Bagging on Trees with Random Forests
+Compare Bagging on Trees with Random Forests
@@ -1880,15 +1946,67 @@ np.sum(y_pred =
-
Feature Importance
+Bootstrap with Random Forests Instead of a Single Tree
-Example will be added here.
+
+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.ensemble import RandomForestRegressor
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = RandomForestRegressor()
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
-
Boosting, a Bird'e Eye
+Boosting, a Bird'e Eye
The basic idea is to combine weak classifiers in order to create a good
@@ -1905,7 +2023,7 @@ them with a factor.
-
Adaptive boosting: AdaBoost, Basic Algorithm
+Adaptive boosting: AdaBoost, Basic Algorithm
The algorithm here is rather straightforward. Assume that our weak
@@ -1927,7 +2045,7 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
-
Basic Steps of AdaBoost
+Basic Steps of AdaBoost
With the above definitions we are now ready to set up the algorithm for AdaBoost.
@@ -1961,18 +2079,18 @@ individually at each steps. The obersvations which were misclassified
at iteration \( m-1 \) have a weight which is larger than those which were
classified properly. As this proceeds, the observations which were
difficult to classifiy correctly are given a larger influence. Each
-new classificatio step \( m \) is then forced to concentrate on those
+new classification step \( m \) is then forced to concentrate on those
observations that are missed in the previous iterations.
-
Figure to Illustrate the Iterative Classification Process
+Figure to Illustrate the Iterative Classification Process
-
AdaBoost Examples
+AdaBoost Examples
Using Scikit-Learn it is easy to appply the adaptive boosting algorithm, as done here.
@@ -2015,7 +2133,7 @@ plt.show()
-
Gradient boosting: Basics
+Gradient boosting: Basics
Gradient boosting is again a similar technique to Adapative boosting,
@@ -2030,7 +2148,7 @@ function was the least squares function.
-
Gradient Boosting, algorithm
+Gradient Boosting, algorithm
Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard least squares function
@@ -2056,7 +2174,7 @@ The way we proceed in an iterative fashion is to
-
Gradient Boosting, Examples
+Gradient Boosting, Examples
@@ -2146,7 +2264,7 @@ plt.show()
-
Gradient Boots with Early Stopping
+Gradient Boots with Early Stopping
@@ -2210,7 +2328,7 @@ error_going_up = XGBoost: Extreme Gradient Boosting
+XGBoost: Extreme Gradient Boosting
XGBoost or Extreme Gradient
@@ -2231,7 +2349,7 @@ It is now the algorithm which wins essentially all ML competitions!!!
-
Xgboost on the Cancer Data
+Xgboost on the Cancer Data
diff --git a/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb b/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
index b0eaa13f3..59ba61dcb 100644
--- a/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
+++ b/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
@@ -10,7 +10,7 @@
" \n",
"**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n",
"\n",
- "Date: **Nov 6, 2019**\n",
+ "Date: **Nov 7, 2019**\n",
"\n",
"Copyright 1999-2019, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license\n",
"\n",
@@ -603,12 +603,13 @@
"metadata": {},
"source": [
"## Algorithms for Setting up Decision Trees\n",
+ "\n",
"Two algorithms stand out in the set up of decision trees:\n",
"1. The CART (Classification And Regression Tree) algorithm for both classification and regression\n",
"\n",
"2. The ID3 algorithm based on the computation of the information gain for classification\n",
"\n",
- "We discuss both algorithms with applications here. The popular library -Scikit-Learn_ uses the CART algorithm. For classification problems you can use either the **gini** index or the **entropy** to split a tree in two branches.\n",
+ "We discuss both algorithms with applications here. The popular library **Scikit-Learn** uses the CART algorithm. For classification problems you can use either the **gini** index or the **entropy** to split a tree in two branches.\n",
"\n",
"## The CART algorithm for Classification\n",
"\n",
@@ -1397,7 +1398,7 @@
"However, by aggregating many decision trees, using methods like bagging, random forests, and boosting, the predictive performance of trees can be substantially improved. \n",
"\n",
"\n",
- "## From a Single Tree to Many Trees, Meet the Jungle of Methods\n",
+ "## Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods\n",
"\n",
"As stated above and seen in many of the examples discussed here about\n",
"a single decision tree, we often end up overfitting our training\n",
@@ -1412,7 +1413,7 @@
"\n",
"3. Random forests\n",
"\n",
- "4. Boosting methods\n",
+ "4. Boosting methods, from adaptive to Extreme Gradient Boosting (XGBoost)\n",
"\n",
"We discuss these methods here.\n",
"\n",
@@ -1726,6 +1727,75 @@
"plt.show()"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Making our own Bagging with Bootstrap"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "from sklearn.pipeline import make_pipeline\n",
+ "from sklearn.utils import resample\n",
+ "from sklearn.tree import DecisionTreeRegressor\n",
+ "\n",
+ "\n",
+ "np.random.seed(2018)\n",
+ "\n",
+ "n = 40\n",
+ "n_boostraps = 100\n",
+ "maxdegree = 14\n",
+ "\n",
+ "# Make data set.\n",
+ "x = np.linspace(-3, 3, n).reshape(-1, 1)\n",
+ "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n",
+ "error = np.zeros(maxdegree)\n",
+ "bias = np.zeros(maxdegree)\n",
+ "variance = np.zeros(maxdegree)\n",
+ "polydegree = np.zeros(maxdegree)\n",
+ "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
+ "\n",
+ "from sklearn.preprocessing import StandardScaler\n",
+ "scaler = StandardScaler()\n",
+ "scaler.fit(X_train)\n",
+ "X_train_scaled = scaler.transform(X_train)\n",
+ "X_test_scaled = scaler.transform(X_test)\n",
+ "\n",
+ "for degree in range(maxdegree):\n",
+ " model = DecisionTreeRegressor(max_depth=5) \n",
+ " y_pred = np.empty((y_test.shape[0], n_boostraps))\n",
+ " for i in range(n_boostraps):\n",
+ " x_, y_ = resample(X_train_scaled, y_train)\n",
+ " model.fit(x_, y_)\n",
+ " y_pred[:, i] = model.predict(X_test_scaled).ravel()\n",
+ "\n",
+ " polydegree[degree] = degree\n",
+ " error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )\n",
+ " bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )\n",
+ " variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )\n",
+ " print('Polynomial degree:', degree)\n",
+ " print('Error:', error[degree])\n",
+ " print('Bias^2:', bias[degree])\n",
+ " print('Var:', variance[degree])\n",
+ " print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))\n",
+ "\n",
+ "plt.plot(polydegree, error, label='Error')\n",
+ "plt.plot(polydegree, bias, label='bias')\n",
+ "plt.plot(polydegree, variance, label='Variance')\n",
+ "plt.legend()\n",
+ "plt.show()"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -1802,7 +1872,7 @@
},
{
"cell_type": "code",
- "execution_count": 24,
+ "execution_count": 25,
"metadata": {
"collapsed": false
},
@@ -1867,7 +1937,6 @@
"import scikitplot as skplt\n",
"y_pred = Random_Forest_model.predict(X_test_scaled)\n",
"skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)\n",
- "#\n",
"plt.show()\n",
"y_probas = Random_Forest_model.predict_proba(X_test_scaled)\n",
"skplt.metrics.plot_roc(y_test, y_probas)\n",
@@ -1885,7 +1954,7 @@
},
{
"cell_type": "code",
- "execution_count": 25,
+ "execution_count": 26,
"metadata": {
"collapsed": false
},
@@ -1898,7 +1967,7 @@
},
{
"cell_type": "code",
- "execution_count": 26,
+ "execution_count": 27,
"metadata": {
"collapsed": false
},
@@ -1917,11 +1986,75 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "## Feature Importance\n",
+ "## Bootstrap with Random Forests Instead of a Single Tree"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
"\n",
- "Example will be added here.\n",
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "from sklearn.pipeline import make_pipeline\n",
+ "from sklearn.utils import resample\n",
+ "from sklearn.ensemble import RandomForestRegressor\n",
"\n",
+ "np.random.seed(2018)\n",
"\n",
+ "n = 40\n",
+ "n_boostraps = 100\n",
+ "maxdegree = 14\n",
+ "\n",
+ "# Make data set.\n",
+ "x = np.linspace(-3, 3, n).reshape(-1, 1)\n",
+ "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n",
+ "error = np.zeros(maxdegree)\n",
+ "bias = np.zeros(maxdegree)\n",
+ "variance = np.zeros(maxdegree)\n",
+ "polydegree = np.zeros(maxdegree)\n",
+ "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
+ "\n",
+ "from sklearn.preprocessing import StandardScaler\n",
+ "scaler = StandardScaler()\n",
+ "scaler.fit(X_train)\n",
+ "X_train_scaled = scaler.transform(X_train)\n",
+ "X_test_scaled = scaler.transform(X_test)\n",
+ "\n",
+ "for degree in range(maxdegree):\n",
+ " model = RandomForestRegressor()\n",
+ " y_pred = np.empty((y_test.shape[0], n_boostraps))\n",
+ " for i in range(n_boostraps):\n",
+ " x_, y_ = resample(X_train_scaled, y_train)\n",
+ " model.fit(x_, y_)\n",
+ " y_pred[:, i] = model.predict(X_test_scaled).ravel()\n",
+ "\n",
+ " polydegree[degree] = degree\n",
+ " error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )\n",
+ " bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )\n",
+ " variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )\n",
+ " print('Polynomial degree:', degree)\n",
+ " print('Error:', error[degree])\n",
+ " print('Bias^2:', bias[degree])\n",
+ " print('Var:', variance[degree])\n",
+ " print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))\n",
+ "\n",
+ "plt.plot(polydegree, error, label='Error')\n",
+ "plt.plot(polydegree, bias, label='bias')\n",
+ "plt.plot(polydegree, variance, label='Variance')\n",
+ "plt.legend()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
"## Boosting, a Bird'e Eye\n",
"\n",
"The basic idea is to combine weak classifiers in order to create a good\n",
@@ -2003,7 +2136,7 @@
"at iteration $m-1$ have a weight which is larger than those which were\n",
"classified properly. As this proceeds, the observations which were\n",
"difficult to classifiy correctly are given a larger influence. Each\n",
- "new classificatio step $m$ is then forced to concentrate on those\n",
+ "new classification step $m$ is then forced to concentrate on those\n",
"observations that are missed in the previous iterations.\n",
"\n",
"## Figure to Illustrate the Iterative Classification Process\n",
@@ -2016,7 +2149,7 @@
},
{
"cell_type": "code",
- "execution_count": 27,
+ "execution_count": 29,
"metadata": {
"collapsed": false
},
@@ -2106,7 +2239,7 @@
},
{
"cell_type": "code",
- "execution_count": 28,
+ "execution_count": 30,
"metadata": {
"collapsed": false
},
@@ -2205,7 +2338,7 @@
},
{
"cell_type": "code",
- "execution_count": 29,
+ "execution_count": 31,
"metadata": {
"collapsed": false
},
@@ -2295,7 +2428,7 @@
},
{
"cell_type": "code",
- "execution_count": 30,
+ "execution_count": 32,
"metadata": {
"collapsed": false
},
diff --git a/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz b/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz
index 8efd3d870..10e484876 100644
Binary files a/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz and b/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz differ
diff --git a/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf b/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf
index 4556a78e2..c5c80b215 100644
Binary files a/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf and b/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf differ
diff --git a/doc/src/DecisionTrees/DecisionTrees.do.txt b/doc/src/DecisionTrees/DecisionTrees.do.txt
index 3d35629da..8926c0e71 100644
--- a/doc/src/DecisionTrees/DecisionTrees.do.txt
+++ b/doc/src/DecisionTrees/DecisionTrees.do.txt
@@ -466,11 +466,12 @@ os.system(cmd)
!split
===== Algorithms for Setting up Decision Trees =====
+
Two algorithms stand out in the set up of decision trees:
o The CART (Classification And Regression Tree) algorithm for both classification and regression
o The ID3 algorithm based on the computation of the information gain for classification
-We discuss both algorithms with applications here. The popular library -Scikit-Learn_ uses the CART algorithm. For classification problems you can use either the _gini_ index or the _entropy_ to split a tree in two branches.
+We discuss both algorithms with applications here. The popular library _Scikit-Learn_ uses the CART algorithm. For classification problems you can use either the _gini_ index or the _entropy_ to split a tree in two branches.
!split
===== The CART algorithm for Classification =====
@@ -1148,7 +1149,7 @@ However, by aggregating many decision trees, using methods like bagging, random
!split
-===== From a Single Tree to Many Trees, Meet the Jungle of Methods =====
+===== Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods =====
As stated above and seen in many of the examples discussed here about
a single decision tree, we often end up overfitting our training
@@ -1160,7 +1161,7 @@ machine learning algorithms or just use one of them to construct forests and jun
o Voting classifiers
o Bagging and Pasting
o Random forests
-o Boosting methods
+o Boosting methods, from adaptive to Extreme Gradient Boosting (XGBoost)
We discuss these methods here.
@@ -1389,6 +1390,66 @@ plt.show()
!ec
+!split
+===== Making our own Bagging with Bootstrap =====
+!bc pycod
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
+
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = DecisionTreeRegressor(max_depth=5)
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+!ec
+
+
+
!split
===== Random forests =====
@@ -1505,7 +1566,6 @@ print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Ran
import scikitplot as skplt
y_pred = Random_Forest_model.predict(X_test_scaled)
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
-#
plt.show()
y_probas = Random_Forest_model.predict_proba(X_test_scaled)
skplt.metrics.plot_roc(y_test, y_probas)
@@ -1539,9 +1599,66 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
!split
-===== Feature Importance =====
+===== Bootstrap with Random Forests Instead of a Single Tree =====
+
+!bc pycod
+
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.ensemble import RandomForestRegressor
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = RandomForestRegressor()
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
+!ec
-Example will be added here.
!split
@@ -1601,7 +1718,7 @@ individually at each steps. The obersvations which were misclassified
at iteration $m-1$ have a weight which is larger than those which were
classified properly. As this proceeds, the observations which were
difficult to classifiy correctly are given a larger influence. Each
-new classificatio step $m$ is then forced to concentrate on those
+new classification step $m$ is then forced to concentrate on those
observations that are missed in the previous iterations.
!split
diff --git a/doc/src/DecisionTrees/Programs/bootstrap.py b/doc/src/DecisionTrees/Programs/bootstrap.py
new file mode 100644
index 000000000..4f6dd3ae8
--- /dev/null
+++ b/doc/src/DecisionTrees/Programs/bootstrap.py
@@ -0,0 +1,56 @@
+
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
+
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = DecisionTreeRegressor(max_depth=5)
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
diff --git a/doc/src/DecisionTrees/Programs/rfbootstrap.py b/doc/src/DecisionTrees/Programs/rfbootstrap.py
new file mode 100644
index 000000000..824862286
--- /dev/null
+++ b/doc/src/DecisionTrees/Programs/rfbootstrap.py
@@ -0,0 +1,54 @@
+
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.ensemble import RandomForestRegressor
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = RandomForestRegressor()
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled)#.ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+