-
@@ -223,7 +232,7 @@ MathJax.Hub.Config({
@@ -219,6 +228,9 @@ plt.show()
@@ -249,6 +258,9 @@ voting_clf.fit(X_train, y_train)
-Random forests provide an improvement over bagged trees by way of a -small tweak that decorrelates the trees. + +
from sklearn.model_selection import train_test_split
+from sklearn.datasets import make_moons
+
+X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
+X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.ensemble import VotingClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.svm import SVC
+
+log_clf = LogisticRegression(random_state=42)
+rnd_clf = RandomForestClassifier(random_state=42)
+svm_clf = SVC(random_state=42)
+
+voting_clf = VotingClassifier(
+ estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
+ voting='hard')
+voting_clf.fit(X_train, y_train)
+-As in bagging, we build a -number of decision trees on bootstrapped training samples. But when -building these decision trees, each time a split in a tree is -considered, a random sample of \( m \) predictors is chosen as split -candidates from the full set of \( p \) predictors. The split is allowed to -use only one of those \( m \) predictors. + +
from sklearn.metrics import accuracy_score
+
+for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
+ clf.fit(X_train, y_train)
+ y_pred = clf.predict(X_test)
+ print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
+-A fresh sample of \( m \) predictors is -taken at each split, and typically we choose -$$ -m\approx \sqrt{p}. -$$ + +
log_clf = LogisticRegression(random_state=42)
+rnd_clf = RandomForestClassifier(random_state=42)
+svm_clf = SVC(probability=True, random_state=42)
+voting_clf = VotingClassifier(
+ estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
+ voting='soft')
+voting_clf.fit(X_train, y_train)
+-In building a random forest, at -each split in the tree, the algorithm is not even allowed to consider -a majority of the available predictors. -
-The reason for this is rather clever. Suppose that there is one very -strong predictor in the data set, along with a number of other -moderately strong predictors. Then in the collection of bagged -variable importance random forest trees, most or all of the trees will -use this strong predictor in the top split. Consequently, all of the -bagged trees will look quite similar to each other. Hence the -predictions from the bagged trees will be highly correlated. -Unfortunately, averaging many highly correlated quantities does not -lead to as large of a reduction in variance as averaging many -uncorrelated quanti- ties. In particular, this means that bagging will -not lead to a substantial reduction in variance over a single tree in -this setting. + +
from sklearn.metrics import accuracy_score
+for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
+ clf.fit(X_train, y_train)
+ y_pred = clf.predict(X_test)
+ print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
+
@@ -241,6 +264,9 @@ this setting.
-
from sklearn.ensemble import RandomForestClassifier
-from sklearn.preprocessing import LabelEncoder
-from sklearn.model_selection import cross_validate
-# Data set not specificied
-X = dataset.XXX
-Y = dataset.YYY
-#Instantiate the model with 100 trees and entropy as splitting criteria
-Random_Forest_model = RandomForestClassifier(n_estimators=100,criterion="entropy")
-#Cross validation
-accuracy = cross_validate(Random_Forest_model,X,Y,cv=10)['test_score']
+from sklearn.ensemble import BaggingClassifier
+from sklearn.tree import DecisionTreeClassifier
+
+bag_clf = BaggingClassifier(
+ DecisionTreeClassifier(random_state=42), n_estimators=500,
+ max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)
+bag_clf.fit(X_train, y_train)
+y_pred = bag_clf.predict(X_test)
+
+
+
+
+
from sklearn.metrics import accuracy_score
+print(accuracy_score(y_test, y_pred))
+
+
+
+
+
tree_clf = DecisionTreeClassifier(random_state=42)
+tree_clf.fit(X_train, y_train)
+y_pred_tree = tree_clf.predict(X_test)
+print(accuracy_score(y_test, y_pred_tree))
+
+
+
+
+
from matplotlib.colors import ListedColormap
+
+def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):
+ x1s = np.linspace(axes[0], axes[1], 100)
+ x2s = np.linspace(axes[2], axes[3], 100)
+ x1, x2 = np.meshgrid(x1s, x2s)
+ X_new = np.c_[x1.ravel(), x2.ravel()]
+ y_pred = clf.predict(X_new).reshape(x1.shape)
+ custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
+ plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
+ if contour:
+ custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
+ plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
+ plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha)
+ plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha)
+ plt.axis(axes)
+ plt.xlabel(r"$x_1$", fontsize=18)
+ plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
+plt.figure(figsize=(11,4))
+plt.subplot(121)
+plot_decision_boundary(tree_clf, X, y)
+plt.title("Decision Tree", fontsize=14)
+plt.subplot(122)
+plot_decision_boundary(bag_clf, X, y)
+plt.title("Decision Trees with Bagging", fontsize=14)
+plt.show()
@@ -214,6 +266,9 @@ accuracy = cross_validate(Random_Forest_mode
+Random forests provide an improvement over bagged trees by way of a +small tweak that decorrelates the trees. - -
from sklearn.model_selection import train_test_split
-from sklearn.datasets import make_moons
-
-X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
-X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.ensemble import VotingClassifier
-from sklearn.linear_model import LogisticRegression
-from sklearn.svm import SVC
-
-log_clf = LogisticRegression(random_state=42)
-rnd_clf = RandomForestClassifier(random_state=42)
-svm_clf = SVC(random_state=42)
-
-voting_clf = VotingClassifier(
- estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
- voting='hard')
-voting_clf.fit(X_train, y_train)
-+As in bagging, we build a +number of decision trees on bootstrapped training samples. But when +building these decision trees, each time a split in a tree is +considered, a random sample of \( m \) predictors is chosen as split +candidates from the full set of \( p \) predictors. The split is allowed to +use only one of those \( m \) predictors. - -
from sklearn.metrics import accuracy_score
-
-for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
- clf.fit(X_train, y_train)
- y_pred = clf.predict(X_test)
- print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
-+A fresh sample of \( m \) predictors is +taken at each split, and typically we choose - -
log_clf = LogisticRegression(random_state=42)
-rnd_clf = RandomForestClassifier(random_state=42)
-svm_clf = SVC(probability=True, random_state=42)
+$$
+m\approx \sqrt{p}.
+$$
-voting_clf = VotingClassifier(
- estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
- voting='soft')
-voting_clf.fit(X_train, y_train)
-+In building a random forest, at +each split in the tree, the algorithm is not even allowed to consider +a majority of the available predictors. - -
from sklearn.metrics import accuracy_score
+
+The reason for this is rather clever. Suppose that there is one very
+strong predictor in the data set, along with a number of other
+moderately strong predictors. Then in the collection of bagged
+variable importance random forest trees, most or all of the trees will
+use this strong predictor in the top split. Consequently, all of the
+bagged trees will look quite similar to each other. Hence the
+predictions from the bagged trees will be highly correlated.
+Unfortunately, averaging many highly correlated quantities does not
+lead to as large of a reduction in variance as averaging many
+uncorrelated quanti- ties. In particular, this means that bagging will
+not lead to a substantial reduction in variance over a single tree in
+this setting.
-for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
- clf.fit(X_train, y_train)
- y_pred = clf.predict(X_test)
- print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
-
@@ -253,6 +248,9 @@ voting_clf.fit(X_train, y_train)
-
from sklearn.ensemble import BaggingClassifier
-from sklearn.tree import DecisionTreeClassifier
-
-bag_clf = BaggingClassifier(
- DecisionTreeClassifier(random_state=42), n_estimators=500,
- max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)
-bag_clf.fit(X_train, y_train)
-y_pred = bag_clf.predict(X_test)
-- - -
from sklearn.metrics import accuracy_score
-print(accuracy_score(y_test, y_pred))
-- - -
tree_clf = DecisionTreeClassifier(random_state=42)
-tree_clf.fit(X_train, y_train)
-y_pred_tree = tree_clf.predict(X_test)
-print(accuracy_score(y_test, y_pred_tree))
-- - -
from matplotlib.colors import ListedColormap
-
-def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):
- x1s = np.linspace(axes[0], axes[1], 100)
- x2s = np.linspace(axes[2], axes[3], 100)
- x1, x2 = np.meshgrid(x1s, x2s)
- X_new = np.c_[x1.ravel(), x2.ravel()]
- y_pred = clf.predict(X_new).reshape(x1.shape)
- custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
- plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
- if contour:
- custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
- plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
- plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha)
- plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha)
- plt.axis(axes)
- plt.xlabel(r"$x_1$", fontsize=18)
- plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
-plt.figure(figsize=(11,4))
-plt.subplot(121)
-plot_decision_boundary(tree_clf, X, y)
-plt.title("Decision Tree", fontsize=14)
-plt.subplot(122)
-plot_decision_boundary(bag_clf, X, y)
-plt.title("Decision Trees with Bagging", fontsize=14)
-plt.show()
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+# Data set not specificied
+X = dataset.XXX
+Y = dataset.YYY
+#Instantiate the model with 100 trees and entropy as splitting criteria
+Random_Forest_model = RandomForestClassifier(n_estimators=100,criterion="entropy")
+#Cross validation
+accuracy = cross_validate(Random_Forest_model,X,Y,cv=10)['test_score']
@@ -255,6 +221,9 @@ plt.show()
-
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs038.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs038.html index 52e58f32b..c3f2f82cd 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs038.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs038.html @@ -47,52 +47,56 @@ Automatically generated HTML file from DocOnce source 2, None, '___sec1'), - ('A typical Decision Tree with its pertinent Jargon, Regeression ' - 'Problem', - 2, - None, - '___sec2'), - ('General Features', 2, None, '___sec3'), - ('How do we set it up?', 2, None, '___sec4'), - ('Decision trees and Regression', 2, None, '___sec5'), - ('Building a tree, regression', 2, None, '___sec6'), + ('General Features', 2, None, '___sec2'), + ('How do we set it up?', 2, None, '___sec3'), + ('Decision trees and Regression', 2, None, '___sec4'), + ('Building a tree, regression', 2, None, '___sec5'), ('A top-down approach, recursive binary splitting', 2, None, - '___sec7'), - ('Making a tree', 2, None, '___sec8'), - ('Pruning the tree', 2, None, '___sec9'), - ('Cost complexity pruning', 2, None, '___sec10'), - ('Schematic Regression Procedure', 2, None, '___sec11'), - ('A Classification Tree', 2, None, '___sec12'), - ('Growing a classification tree', 2, None, '___sec13'), - ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Visualizing the Tree, Classification', 2, None, '___sec15'), - ('Visualizing the Tree, The Moons', 2, None, '___sec16'), - ('Computing the Gini index', 2, None, '___sec17'), - ('Simple Python Code to read in Data', 2, None, '___sec18'), - ('Computing the Gini Factor', 2, None, '___sec19'), - ('Entropy and the ID3 algorithm', 2, None, '___sec20'), - ('Implementing the ID3 Algorithm', 2, None, '___sec21'), + '___sec6'), + ('Making a tree', 2, None, '___sec7'), + ('Pruning the tree', 2, None, '___sec8'), + ('Cost complexity pruning', 2, None, '___sec9'), + ('Schematic Regression Procedure', 2, None, '___sec10'), + ('A Classification Tree', 2, None, '___sec11'), + ('Growing a classification tree', 2, None, '___sec12'), + ('Classification tree, how to split nodes', 2, None, '___sec13'), + ('Visualizing the Tree, Classification', 2, None, '___sec14'), + ('Visualizing the Tree, The Moons', 2, None, '___sec15'), + ('Computing the Gini index', 2, None, '___sec16'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + '___sec17'), + ('Computing the Gini Factor', 2, None, '___sec18'), + ('Entropy and the ID3 algorithm', 2, None, '___sec19'), + ('Implementing the ID3 Algorithm', 2, None, '___sec20'), ('Cancer Data again now with Decision Trees and other Methods', 2, None, - '___sec22'), - ('Another example, the moons again', 2, None, '___sec23'), - ('Playing around with regions', 2, None, '___sec24'), - ('Regression trees', 2, None, '___sec25'), - ('Final regressor code', 2, None, '___sec26'), - ('Pros and cons of trees, pros', 2, None, '___sec27'), - ('Disadvantages', 2, None, '___sec28'), - ('Bagging', 2, None, '___sec29'), - ('More bagging', 2, None, '___sec30'), - ('Simple example, head or tail', 2, None, '___sec31'), - ('Bagging Example', 2, None, '___sec32'), - ('Random forests', 2, None, '___sec33'), - ('A simple scikit-learn example', 2, None, '___sec34'), - ('Please, not the moons again!', 2, None, '___sec35'), - ('Bagging examples', 2, None, '___sec36'), - ('Then random forests', 2, None, '___sec37')]} + '___sec21'), + ('Another example, the moons again', 2, None, '___sec22'), + ('Playing around with regions', 2, None, '___sec23'), + ('Regression trees', 2, None, '___sec24'), + ('Final regressor code', 2, None, '___sec25'), + ('Pros and cons of trees, pros', 2, None, '___sec26'), + ('Disadvantages', 2, None, '___sec27'), + ('Bagging', 2, None, '___sec28'), + ('More bagging', 2, None, '___sec29'), + ('Simple Voting Example, head or tail', 2, None, '___sec30'), + ('Using the Voting Classifier', 2, None, '___sec31'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + '___sec32'), + ('Now Bagging', 2, None, '___sec33'), + ('Random forests', 2, None, '___sec34'), + ('A simple scikit-learn example', 2, None, '___sec35'), + ('Then random forests', 2, None, '___sec36'), + ('Feature Importance', 2, None, '___sec37'), + ('Boosting: AdaBoost', 2, None, '___sec38'), + ('Gradient Boosting', 2, None, '___sec39')]} end of tocinfo --> @@ -132,42 +136,44 @@ MathJax.Hub.Config({ @@ -183,27 +189,37 @@ MathJax.Hub.Config({ -
-
bag_clf = BaggingClassifier(
- DecisionTreeClassifier(splitter="random", max_leaf_nodes=16, random_state=42),
- n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1, random_state=42)
+try:
+ from sklearn.datasets import fetch_openml
+ mnist = fetch_openml('mnist_784', version=1)
+ mnist.target = mnist.target.astype(np.int64)
+except ImportError:
+ from sklearn.datasets import fetch_mldata
+ mnist = fetch_mldata('MNIST original')
+
+rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
+rnd_clf.fit(mnist["data"], mnist["target"])
+
+def plot_digit(data):
+ image = data.reshape(28, 28)
+ plt.imshow(image, cmap = mpl.cm.hot,
+ interpolation="nearest")
+ plt.axis("off")
+
+plot_digit(rnd_clf.feature_importances_)
+
+cbar = plt.colorbar(ticks=[rnd_clf.feature_importances_.min(), rnd_clf.feature_importances_.max()])
+cbar.ax.set_yticklabels(['Not important', 'Very important'])
+
+#save_fig("mnist_feature_importance_plot")
+plt.show()
-
-
-
bag_clf.fit(X_train, y_train)
-y_pred = bag_clf.predict(X_test)
-from sklearn.ensemble import RandomForestClassifier
-rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)
-rnd_clf.fit(X_train, y_train)
-y_pred_rf = rnd_clf.predict(X_test)
-np.sum(y_pred == y_pred_rf) / len(y_pred)
-
-
-
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-bs.html b/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
index 04d083734..d91a75045 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
@@ -84,13 +84,19 @@ Automatically generated HTML file from DocOnce source
('Disadvantages', 2, None, '___sec27'),
('Bagging', 2, None, '___sec28'),
('More bagging', 2, None, '___sec29'),
- ('Simple example, head or tail', 2, None, '___sec30'),
- ('Bagging Example', 2, None, '___sec31'),
- ('Random forests', 2, None, '___sec32'),
- ('A simple scikit-learn example', 2, None, '___sec33'),
- ('Please, not the moons again!', 2, None, '___sec34'),
- ('Bagging examples', 2, None, '___sec35'),
- ('Then random forests', 2, None, '___sec36')]}
+ ('Simple Voting Example, head or tail', 2, None, '___sec30'),
+ ('Using the Voting Classifier', 2, None, '___sec31'),
+ ('Please, not the moons again! Voting and Bagging',
+ 2,
+ None,
+ '___sec32'),
+ ('Now Bagging', 2, None, '___sec33'),
+ ('Random forests', 2, None, '___sec34'),
+ ('A simple scikit-learn example', 2, None, '___sec35'),
+ ('Then random forests', 2, None, '___sec36'),
+ ('Feature Importance', 2, None, '___sec37'),
+ ('Boosting: AdaBoost', 2, None, '___sec38'),
+ ('Gradient Boosting', 2, None, '___sec39')]}
end of tocinfo -->
@@ -158,13 +164,16 @@ MathJax.Hub.Config({
-
@@ -223,7 +232,7 @@ MathJax.Hub.Config({
-
@@ -1457,7 +1457,7 @@ predictor, averaged over all \( B \) trees.
@@ -1478,7 +1478,7 @@ plt.show()
@@ -1530,72 +1530,7 @@ voting_clf.fit(X_train, y_train)
-Random forests provide an improvement over bagged trees by way of a
-small tweak that decorrelates the trees.
-
-
-As in bagging, we build a
-number of decision trees on bootstrapped training samples. But when
-building these decision trees, each time a split in a tree is
-considered, a random sample of \( m \) predictors is chosen as split
-candidates from the full set of \( p \) predictors. The split is allowed to
-use only one of those \( m \) predictors.
-
-
-A fresh sample of \( m \) predictors is
-taken at each split, and typically we choose
-
-
-In building a random forest, at
-each split in the tree, the algorithm is not even allowed to consider
-a majority of the available predictors.
-
-
-The reason for this is rather clever. Suppose that there is one very
-strong predictor in the data set, along with a number of other
-moderately strong predictors. Then in the collection of bagged
-variable importance random forest trees, most or all of the trees will
-use this strong predictor in the top split. Consequently, all of the
-bagged trees will look quite similar to each other. Hence the
-predictions from the bagged trees will be highly correlated.
-Unfortunately, averaging many highly correlated quantities does not
-lead to as large of a reduction in variance as averaging many
-uncorrelated quanti- ties. In particular, this means that bagging will
-not lead to a substantial reduction in variance over a single tree in
-this setting.
-
-
-
-
@@ -1654,7 +1589,7 @@ voting_clf.fit(X_train, y_train)
@@ -1715,6 +1650,71 @@ plt.show()
+Random forests provide an improvement over bagged trees by way of a
+small tweak that decorrelates the trees.
+
+
+As in bagging, we build a
+number of decision trees on bootstrapped training samples. But when
+building these decision trees, each time a split in a tree is
+considered, a random sample of \( m \) predictors is chosen as split
+candidates from the full set of \( p \) predictors. The split is allowed to
+use only one of those \( m \) predictors.
+
+
+A fresh sample of \( m \) predictors is
+taken at each split, and typically we choose
+
+
+In building a random forest, at
+each split in the tree, the algorithm is not even allowed to consider
+a majority of the available predictors.
+
+
+The reason for this is rather clever. Suppose that there is one very
+strong predictor in the data set, along with a number of other
+moderately strong predictors. Then in the collection of bagged
+variable importance random forest trees, most or all of the trees will
+use this strong predictor in the top split. Consequently, all of the
+bagged trees will look quite similar to each other. Hence the
+predictions from the bagged trees will be highly correlated.
+Unfortunately, averaging many highly correlated quantities does not
+lead to as large of a reduction in variance as averaging many
+uncorrelated quanti- ties. In particular, this means that bagging will
+not lead to a substantial reduction in variance over a single tree in
+this setting.
+
+
+
+
@@ -1738,6 +1738,172 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
+
+
+
+
+
+
+
+
+Simple example, head or tail
+Simple Voting Example, head or tail
Bagging Example
+Using the Voting Classifier
Random forests
-
-
-$$
-m\approx \sqrt{p}.
-$$
-
-
-A simple scikit-learn example
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.preprocessing import LabelEncoder
-from sklearn.model_selection import cross_validate
-# Data set not specificied
-X = dataset.XXX
-Y = dataset.YYY
-#Instantiate the model with 100 trees and entropy as splitting criteria
-Random_Forest_model = RandomForestClassifier(n_estimators=100,criterion="entropy")
-#Cross validation
-accuracy = cross_validate(Random_Forest_model,X,Y,cv=10)['test_score']
-
Please, not the moons again!
+Please, not the moons again! Voting and Bagging
Bagging examples
+Now Bagging
Random forests
+
+
+$$
+m\approx \sqrt{p}.
+$$
+
+
+A simple scikit-learn example
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+# Data set not specificied
+X = dataset.XXX
+Y = dataset.YYY
+#Instantiate the model with 100 trees and entropy as splitting criteria
+Random_Forest_model = RandomForestClassifier(n_estimators=100,criterion="entropy")
+#Cross validation
+accuracy = cross_validate(Random_Forest_model,X,Y,cv=10)['test_score']
+
Then random forests
Feature Importance
+
+try:
+ from sklearn.datasets import fetch_openml
+ mnist = fetch_openml('mnist_784', version=1)
+ mnist.target = mnist.target.astype(np.int64)
+except ImportError:
+ from sklearn.datasets import fetch_mldata
+ mnist = fetch_mldata('MNIST original')
+
+rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
+rnd_clf.fit(mnist["data"], mnist["target"])
+
+def plot_digit(data):
+ image = data.reshape(28, 28)
+ plt.imshow(image, cmap = mpl.cm.hot,
+ interpolation="nearest")
+ plt.axis("off")
+
+plot_digit(rnd_clf.feature_importances_)
+
+cbar = plt.colorbar(ticks=[rnd_clf.feature_importances_.min(), rnd_clf.feature_importances_.max()])
+cbar.ax.set_yticklabels(['Not important', 'Very important'])
+
+#save_fig("mnist_feature_importance_plot")
+plt.show()
+
Boosting: AdaBoost
+
+from sklearn.ensemble import AdaBoostClassifier
+
+ada_clf = AdaBoostClassifier(
+ DecisionTreeClassifier(max_depth=1), n_estimators=200,
+ algorithm="SAMME.R", learning_rate=0.5, random_state=42)
+ada_clf.fit(X_train, y_train)
+
+plot_decision_boundary(ada_clf, X, y)
+
+m = len(X_train)
+
+plt.figure(figsize=(11, 4))
+for subplot, learning_rate in ((121, 1), (122, 0.5)):
+ sample_weights = np.ones(m)
+ plt.subplot(subplot)
+ for i in range(5):
+ svm_clf = SVC(kernel="rbf", C=0.05, gamma="auto", random_state=42)
+ svm_clf.fit(X_train, y_train, sample_weight=sample_weights)
+ y_pred = svm_clf.predict(X_train)
+ sample_weights[y_pred != y_train] *= (1 + learning_rate)
+ plot_decision_boundary(svm_clf, X, y, alpha=0.2)
+ plt.title("learning_rate = {}".format(learning_rate), fontsize=16)
+ if subplot == 121:
+ plt.text(-0.7, -0.65, "1", fontsize=14)
+ plt.text(-0.6, -0.10, "2", fontsize=14)
+ plt.text(-0.5, 0.10, "3", fontsize=14)
+ plt.text(-0.4, 0.55, "4", fontsize=14)
+ plt.text(-0.3, 0.90, "5", fontsize=14)
+
+save_fig("boosting_plot")
+plt.show()
+
Gradient Boosting
+np.random.seed(42)
+X = np.random.rand(100, 1) - 0.5
+y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)
+
+from sklearn.tree import DecisionTreeRegressor
+
+tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg1.fit(X, y)
+
+y2 = y - tree_reg1.predict(X)
+tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg2.fit(X, y2)
+
+y3 = y2 - tree_reg2.predict(X)
+tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg3.fit(X, y3)
+
+X_new = np.array([[0.8]])
+y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))
+
+def plot_predictions(regressors, X, y, axes, label=None, style="r-", data_style="b.", data_label=None):
+ x1 = np.linspace(axes[0], axes[1], 500)
+ y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)
+ plt.plot(X[:, 0], y, data_style, label=data_label)
+ plt.plot(x1, y_pred, style, linewidth=2, label=label)
+ if label or data_label:
+ plt.legend(loc="upper center", fontsize=16)
+ plt.axis(axes)
+
+plt.figure(figsize=(11,11))
+
+plt.subplot(321)
+plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h_1(x_1)$", style="g-", data_label="Training set")
+plt.ylabel("$y$", fontsize=16, rotation=0)
+plt.title("Residuals and tree predictions", fontsize=16)
+
+plt.subplot(322)
+plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1)$", data_label="Training set")
+plt.ylabel("$y$", fontsize=16, rotation=0)
+plt.title("Ensemble predictions", fontsize=16)
+
+plt.subplot(323)
+plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_2(x_1)$", style="g-", data_style="k+", data_label="Residuals")
+plt.ylabel("$y - h_1(x_1)$", fontsize=16)
+
+plt.subplot(324)
+plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1)$")
+plt.ylabel("$y$", fontsize=16, rotation=0)
+
+plt.subplot(325)
+plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_3(x_1)$", style="g-", data_style="k+")
+plt.ylabel("$y - h_1(x_1) - h_2(x_1)$", fontsize=16)
+plt.xlabel("$x_1$", fontsize=16)
+
+plt.subplot(326)
+plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$")
+plt.xlabel("$x_1$", fontsize=16)
+plt.ylabel("$y$", fontsize=16, rotation=0)
+
+save_fig("gradient_boosting_plot")
+plt.show()
+
+from sklearn.ensemble import GradientBoostingRegressor
+
+gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0, random_state=42)
+gbrt.fit(X, y)
+
+gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)
+gbrt_slow.fit(X, y)
+
+plt.figure(figsize=(11,4))
+
+plt.subplot(121)
+plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="Ensemble predictions")
+plt.title("learning_rate={}, n_estimators={}".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)
+
+plt.subplot(122)
+plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
+plt.title("learning_rate={}, n_estimators={}".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)
+
+save_fig("gbrt_learning_rate_plot")
+plt.show()
+
-
@@ -1422,7 +1428,7 @@ predictor, averaged over all \( B \) trees.
-
@@ -1442,7 +1448,7 @@ plt.show()
-
@@ -1493,69 +1499,7 @@ voting_clf.fit(X_train, y_train)
-
-Random forests provide an improvement over bagged trees by way of a -small tweak that decorrelates the trees. - -
-As in bagging, we build a -number of decision trees on bootstrapped training samples. But when -building these decision trees, each time a split in a tree is -considered, a random sample of \( m \) predictors is chosen as split -candidates from the full set of \( p \) predictors. The split is allowed to -use only one of those \( m \) predictors. - -
-A fresh sample of \( m \) predictors is -taken at each split, and typically we choose - -$$ -m\approx \sqrt{p}. -$$ - -
-In building a random forest, at -each split in the tree, the algorithm is not even allowed to consider -a majority of the available predictors. - -
-The reason for this is rather clever. Suppose that there is one very -strong predictor in the data set, along with a number of other -moderately strong predictors. Then in the collection of bagged -variable importance random forest trees, most or all of the trees will -use this strong predictor in the top split. Consequently, all of the -bagged trees will look quite similar to each other. Hence the -predictions from the bagged trees will be highly correlated. -Unfortunately, averaging many highly correlated quantities does not -lead to as large of a reduction in variance as averaging many -uncorrelated quanti- ties. In particular, this means that bagging will -not lead to a substantial reduction in variance over a single tree in -this setting. - -
-
-
-
- - -
from sklearn.ensemble import RandomForestClassifier
-from sklearn.preprocessing import LabelEncoder
-from sklearn.model_selection import cross_validate
-# Data set not specificied
-X = dataset.XXX
-Y = dataset.YYY
-#Instantiate the model with 100 trees and entropy as splitting criteria
-Random_Forest_model = RandomForestClassifier(n_estimators=100,criterion="entropy")
-#Cross validation
-accuracy = cross_validate(Random_Forest_model,X,Y,cv=10)['test_score']
-
-
-
-
@@ -1613,7 +1557,7 @@ voting_clf.fit(X_train, y_train)
-
@@ -1674,6 +1618,68 @@ plt.show()
+
+Random forests provide an improvement over bagged trees by way of a +small tweak that decorrelates the trees. + +
+As in bagging, we build a +number of decision trees on bootstrapped training samples. But when +building these decision trees, each time a split in a tree is +considered, a random sample of \( m \) predictors is chosen as split +candidates from the full set of \( p \) predictors. The split is allowed to +use only one of those \( m \) predictors. + +
+A fresh sample of \( m \) predictors is +taken at each split, and typically we choose + +$$ +m\approx \sqrt{p}. +$$ + +
+In building a random forest, at +each split in the tree, the algorithm is not even allowed to consider +a majority of the available predictors. + +
+The reason for this is rather clever. Suppose that there is one very +strong predictor in the data set, along with a number of other +moderately strong predictors. Then in the collection of bagged +variable importance random forest trees, most or all of the trees will +use this strong predictor in the top split. Consequently, all of the +bagged trees will look quite similar to each other. Hence the +predictions from the bagged trees will be highly correlated. +Unfortunately, averaging many highly correlated quantities does not +lead to as large of a reduction in variance as averaging many +uncorrelated quanti- ties. In particular, this means that bagging will +not lead to a substantial reduction in variance over a single tree in +this setting. + +
+
+
+
+ + +
from sklearn.ensemble import RandomForestClassifier
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+# Data set not specificied
+X = dataset.XXX
+Y = dataset.YYY
+#Instantiate the model with 100 trees and entropy as splitting criteria
+Random_Forest_model = RandomForestClassifier(n_estimators=100,criterion="entropy")
+#Cross validation
+accuracy = cross_validate(Random_Forest_model,X,Y,cv=10)['test_score']
+
+
+
@@ -1694,6 +1700,169 @@ y_pred_rf = rnd_clf.predict(X_test) np.sum(y_pred == y_pred_rf) / len(y_pred)
+
+
+
+ + +
try:
+ from sklearn.datasets import fetch_openml
+ mnist = fetch_openml('mnist_784', version=1)
+ mnist.target = mnist.target.astype(np.int64)
+except ImportError:
+ from sklearn.datasets import fetch_mldata
+ mnist = fetch_mldata('MNIST original')
+
+rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
+rnd_clf.fit(mnist["data"], mnist["target"])
+
+def plot_digit(data):
+ image = data.reshape(28, 28)
+ plt.imshow(image, cmap = mpl.cm.hot,
+ interpolation="nearest")
+ plt.axis("off")
+
+plot_digit(rnd_clf.feature_importances_)
+
+cbar = plt.colorbar(ticks=[rnd_clf.feature_importances_.min(), rnd_clf.feature_importances_.max()])
+cbar.ax.set_yticklabels(['Not important', 'Very important'])
+
+#save_fig("mnist_feature_importance_plot")
+plt.show()
+
+
+
+
+ + +
from sklearn.ensemble import AdaBoostClassifier
+
+ada_clf = AdaBoostClassifier(
+ DecisionTreeClassifier(max_depth=1), n_estimators=200,
+ algorithm="SAMME.R", learning_rate=0.5, random_state=42)
+ada_clf.fit(X_train, y_train)
+
+plot_decision_boundary(ada_clf, X, y)
+
+m = len(X_train)
+
+plt.figure(figsize=(11, 4))
+for subplot, learning_rate in ((121, 1), (122, 0.5)):
+ sample_weights = np.ones(m)
+ plt.subplot(subplot)
+ for i in range(5):
+ svm_clf = SVC(kernel="rbf", C=0.05, gamma="auto", random_state=42)
+ svm_clf.fit(X_train, y_train, sample_weight=sample_weights)
+ y_pred = svm_clf.predict(X_train)
+ sample_weights[y_pred != y_train] *= (1 + learning_rate)
+ plot_decision_boundary(svm_clf, X, y, alpha=0.2)
+ plt.title("learning_rate = {}".format(learning_rate), fontsize=16)
+ if subplot == 121:
+ plt.text(-0.7, -0.65, "1", fontsize=14)
+ plt.text(-0.6, -0.10, "2", fontsize=14)
+ plt.text(-0.5, 0.10, "3", fontsize=14)
+ plt.text(-0.4, 0.55, "4", fontsize=14)
+ plt.text(-0.3, 0.90, "5", fontsize=14)
+
+save_fig("boosting_plot")
+plt.show()
+
+
+
+
+ + +
np.random.seed(42)
+X = np.random.rand(100, 1) - 0.5
+y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)
+
+from sklearn.tree import DecisionTreeRegressor
+
+tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg1.fit(X, y)
+
+y2 = y - tree_reg1.predict(X)
+tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg2.fit(X, y2)
+
+y3 = y2 - tree_reg2.predict(X)
+tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg3.fit(X, y3)
+
+X_new = np.array([[0.8]])
+y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))
+
+def plot_predictions(regressors, X, y, axes, label=None, style="r-", data_style="b.", data_label=None):
+ x1 = np.linspace(axes[0], axes[1], 500)
+ y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)
+ plt.plot(X[:, 0], y, data_style, label=data_label)
+ plt.plot(x1, y_pred, style, linewidth=2, label=label)
+ if label or data_label:
+ plt.legend(loc="upper center", fontsize=16)
+ plt.axis(axes)
+
+plt.figure(figsize=(11,11))
+
+plt.subplot(321)
+plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h_1(x_1)$", style="g-", data_label="Training set")
+plt.ylabel("$y$", fontsize=16, rotation=0)
+plt.title("Residuals and tree predictions", fontsize=16)
+
+plt.subplot(322)
+plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1)$", data_label="Training set")
+plt.ylabel("$y$", fontsize=16, rotation=0)
+plt.title("Ensemble predictions", fontsize=16)
+
+plt.subplot(323)
+plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_2(x_1)$", style="g-", data_style="k+", data_label="Residuals")
+plt.ylabel("$y - h_1(x_1)$", fontsize=16)
+
+plt.subplot(324)
+plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1)$")
+plt.ylabel("$y$", fontsize=16, rotation=0)
+
+plt.subplot(325)
+plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_3(x_1)$", style="g-", data_style="k+")
+plt.ylabel("$y - h_1(x_1) - h_2(x_1)$", fontsize=16)
+plt.xlabel("$x_1$", fontsize=16)
+
+plt.subplot(326)
+plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$")
+plt.xlabel("$x_1$", fontsize=16)
+plt.ylabel("$y$", fontsize=16, rotation=0)
+
+save_fig("gradient_boosting_plot")
+plt.show()
+
+from sklearn.ensemble import GradientBoostingRegressor
+
+gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0, random_state=42)
+gbrt.fit(X, y)
+
+gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)
+gbrt_slow.fit(X, y)
+
+plt.figure(figsize=(11,4))
+
+plt.subplot(121)
+plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="Ensemble predictions")
+plt.title("learning_rate={}, n_estimators={}".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)
+
+plt.subplot(122)
+plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
+plt.title("learning_rate={}, n_estimators={}".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)
+
+save_fig("gbrt_learning_rate_plot")
+plt.show()
+diff --git a/doc/pub/DecisionTrees/html/DecisionTrees.html b/doc/pub/DecisionTrees/html/DecisionTrees.html index 5b6adb746..8a7179846 100644 --- a/doc/pub/DecisionTrees/html/DecisionTrees.html +++ b/doc/pub/DecisionTrees/html/DecisionTrees.html @@ -109,13 +109,19 @@ div { text-align: justify; text-justify: inter-word; } ('Disadvantages', 2, None, '___sec27'), ('Bagging', 2, None, '___sec28'), ('More bagging', 2, None, '___sec29'), - ('Simple example, head or tail', 2, None, '___sec30'), - ('Bagging Example', 2, None, '___sec31'), - ('Random forests', 2, None, '___sec32'), - ('A simple scikit-learn example', 2, None, '___sec33'), - ('Please, not the moons again!', 2, None, '___sec34'), - ('Bagging examples', 2, None, '___sec35'), - ('Then random forests', 2, None, '___sec36')]} + ('Simple Voting Example, head or tail', 2, None, '___sec30'), + ('Using the Voting Classifier', 2, None, '___sec31'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + '___sec32'), + ('Now Bagging', 2, None, '___sec33'), + ('Random forests', 2, None, '___sec34'), + ('A simple scikit-learn example', 2, None, '___sec35'), + ('Then random forests', 2, None, '___sec36'), + ('Feature Importance', 2, None, '___sec37'), + ('Boosting: AdaBoost', 2, None, '___sec38'), + ('Gradient Boosting', 2, None, '___sec39')]} end of tocinfo -->
@@ -157,7 +163,7 @@ MathJax.Hub.Config({-
@@ -1427,7 +1433,7 @@ predictor, averaged over all \( B \) trees.
-
@@ -1447,7 +1453,7 @@ plt.show()
-
@@ -1498,69 +1504,7 @@ voting_clf.fit(X_train, y_train)
-
-Random forests provide an improvement over bagged trees by way of a -small tweak that decorrelates the trees. - -
-As in bagging, we build a -number of decision trees on bootstrapped training samples. But when -building these decision trees, each time a split in a tree is -considered, a random sample of \( m \) predictors is chosen as split -candidates from the full set of \( p \) predictors. The split is allowed to -use only one of those \( m \) predictors. - -
-A fresh sample of \( m \) predictors is -taken at each split, and typically we choose - -$$ -m\approx \sqrt{p}. -$$ - -
-In building a random forest, at -each split in the tree, the algorithm is not even allowed to consider -a majority of the available predictors. - -
-The reason for this is rather clever. Suppose that there is one very -strong predictor in the data set, along with a number of other -moderately strong predictors. Then in the collection of bagged -variable importance random forest trees, most or all of the trees will -use this strong predictor in the top split. Consequently, all of the -bagged trees will look quite similar to each other. Hence the -predictions from the bagged trees will be highly correlated. -Unfortunately, averaging many highly correlated quantities does not -lead to as large of a reduction in variance as averaging many -uncorrelated quanti- ties. In particular, this means that bagging will -not lead to a substantial reduction in variance over a single tree in -this setting. - -
-
-
-
- - -
from sklearn.ensemble import RandomForestClassifier
-from sklearn.preprocessing import LabelEncoder
-from sklearn.model_selection import cross_validate
-# Data set not specificied
-X = dataset.XXX
-Y = dataset.YYY
-#Instantiate the model with 100 trees and entropy as splitting criteria
-Random_Forest_model = RandomForestClassifier(n_estimators=100,criterion="entropy")
-#Cross validation
-accuracy = cross_validate(Random_Forest_model,X,Y,cv=10)['test_score']
-
-
-
-
@@ -1618,7 +1562,7 @@ voting_clf.fit(X_train, y_train)
-
@@ -1679,6 +1623,68 @@ plt.show()
+
+Random forests provide an improvement over bagged trees by way of a +small tweak that decorrelates the trees. + +
+As in bagging, we build a +number of decision trees on bootstrapped training samples. But when +building these decision trees, each time a split in a tree is +considered, a random sample of \( m \) predictors is chosen as split +candidates from the full set of \( p \) predictors. The split is allowed to +use only one of those \( m \) predictors. + +
+A fresh sample of \( m \) predictors is +taken at each split, and typically we choose + +$$ +m\approx \sqrt{p}. +$$ + +
+In building a random forest, at +each split in the tree, the algorithm is not even allowed to consider +a majority of the available predictors. + +
+The reason for this is rather clever. Suppose that there is one very +strong predictor in the data set, along with a number of other +moderately strong predictors. Then in the collection of bagged +variable importance random forest trees, most or all of the trees will +use this strong predictor in the top split. Consequently, all of the +bagged trees will look quite similar to each other. Hence the +predictions from the bagged trees will be highly correlated. +Unfortunately, averaging many highly correlated quantities does not +lead to as large of a reduction in variance as averaging many +uncorrelated quanti- ties. In particular, this means that bagging will +not lead to a substantial reduction in variance over a single tree in +this setting. + +
+
+
+
+ + +
from sklearn.ensemble import RandomForestClassifier
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+# Data set not specificied
+X = dataset.XXX
+Y = dataset.YYY
+#Instantiate the model with 100 trees and entropy as splitting criteria
+Random_Forest_model = RandomForestClassifier(n_estimators=100,criterion="entropy")
+#Cross validation
+accuracy = cross_validate(Random_Forest_model,X,Y,cv=10)['test_score']
+
+
+
@@ -1699,6 +1705,169 @@ y_pred_rf = rnd_clf.sum(y_pred == y_pred_rf) / len(y_pred)
+
+
+
+ + +
try:
+ from sklearn.datasets import fetch_openml
+ mnist = fetch_openml('mnist_784', version=1)
+ mnist.target = mnist.target.astype(np.int64)
+except ImportError:
+ from sklearn.datasets import fetch_mldata
+ mnist = fetch_mldata('MNIST original')
+
+rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
+rnd_clf.fit(mnist["data"], mnist["target"])
+
+def plot_digit(data):
+ image = data.reshape(28, 28)
+ plt.imshow(image, cmap = mpl.cm.hot,
+ interpolation="nearest")
+ plt.axis("off")
+
+plot_digit(rnd_clf.feature_importances_)
+
+cbar = plt.colorbar(ticks=[rnd_clf.feature_importances_.min(), rnd_clf.feature_importances_.max()])
+cbar.ax.set_yticklabels(['Not important', 'Very important'])
+
+#save_fig("mnist_feature_importance_plot")
+plt.show()
+
+
+
+
+ + +
from sklearn.ensemble import AdaBoostClassifier
+
+ada_clf = AdaBoostClassifier(
+ DecisionTreeClassifier(max_depth=1), n_estimators=200,
+ algorithm="SAMME.R", learning_rate=0.5, random_state=42)
+ada_clf.fit(X_train, y_train)
+
+plot_decision_boundary(ada_clf, X, y)
+
+m = len(X_train)
+
+plt.figure(figsize=(11, 4))
+for subplot, learning_rate in ((121, 1), (122, 0.5)):
+ sample_weights = np.ones(m)
+ plt.subplot(subplot)
+ for i in range(5):
+ svm_clf = SVC(kernel="rbf", C=0.05, gamma="auto", random_state=42)
+ svm_clf.fit(X_train, y_train, sample_weight=sample_weights)
+ y_pred = svm_clf.predict(X_train)
+ sample_weights[y_pred != y_train] *= (1 + learning_rate)
+ plot_decision_boundary(svm_clf, X, y, alpha=0.2)
+ plt.title("learning_rate = {}".format(learning_rate), fontsize=16)
+ if subplot == 121:
+ plt.text(-0.7, -0.65, "1", fontsize=14)
+ plt.text(-0.6, -0.10, "2", fontsize=14)
+ plt.text(-0.5, 0.10, "3", fontsize=14)
+ plt.text(-0.4, 0.55, "4", fontsize=14)
+ plt.text(-0.3, 0.90, "5", fontsize=14)
+
+save_fig("boosting_plot")
+plt.show()
+
+
+
+
+ + +
np.random.seed(42)
+X = np.random.rand(100, 1) - 0.5
+y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)
+
+from sklearn.tree import DecisionTreeRegressor
+
+tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg1.fit(X, y)
+
+y2 = y - tree_reg1.predict(X)
+tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg2.fit(X, y2)
+
+y3 = y2 - tree_reg2.predict(X)
+tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg3.fit(X, y3)
+
+X_new = np.array([[0.8]])
+y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))
+
+def plot_predictions(regressors, X, y, axes, label=None, style="r-", data_style="b.", data_label=None):
+ x1 = np.linspace(axes[0], axes[1], 500)
+ y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)
+ plt.plot(X[:, 0], y, data_style, label=data_label)
+ plt.plot(x1, y_pred, style, linewidth=2, label=label)
+ if label or data_label:
+ plt.legend(loc="upper center", fontsize=16)
+ plt.axis(axes)
+
+plt.figure(figsize=(11,11))
+
+plt.subplot(321)
+plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h_1(x_1)$", style="g-", data_label="Training set")
+plt.ylabel("$y$", fontsize=16, rotation=0)
+plt.title("Residuals and tree predictions", fontsize=16)
+
+plt.subplot(322)
+plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1)$", data_label="Training set")
+plt.ylabel("$y$", fontsize=16, rotation=0)
+plt.title("Ensemble predictions", fontsize=16)
+
+plt.subplot(323)
+plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_2(x_1)$", style="g-", data_style="k+", data_label="Residuals")
+plt.ylabel("$y - h_1(x_1)$", fontsize=16)
+
+plt.subplot(324)
+plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1)$")
+plt.ylabel("$y$", fontsize=16, rotation=0)
+
+plt.subplot(325)
+plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_3(x_1)$", style="g-", data_style="k+")
+plt.ylabel("$y - h_1(x_1) - h_2(x_1)$", fontsize=16)
+plt.xlabel("$x_1$", fontsize=16)
+
+plt.subplot(326)
+plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$")
+plt.xlabel("$x_1$", fontsize=16)
+plt.ylabel("$y$", fontsize=16, rotation=0)
+
+save_fig("gradient_boosting_plot")
+plt.show()
+
+from sklearn.ensemble import GradientBoostingRegressor
+
+gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0, random_state=42)
+gbrt.fit(X, y)
+
+gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)
+gbrt_slow.fit(X, y)
+
+plt.figure(figsize=(11,4))
+
+plt.subplot(121)
+plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="Ensemble predictions")
+plt.title("learning_rate={}, n_estimators={}".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)
+
+plt.subplot(122)
+plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
+plt.title("learning_rate={}, n_estimators={}".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)
+
+save_fig("gbrt_learning_rate_plot")
+plt.show()
+diff --git a/doc/pub/DecisionTrees/ipynb/DataFiles/cancer.dot b/doc/pub/DecisionTrees/ipynb/DataFiles/cancer.dot index 252087b50..0d1aedaf9 100644 --- a/doc/pub/DecisionTrees/ipynb/DataFiles/cancer.dot +++ b/doc/pub/DecisionTrees/ipynb/DataFiles/cancer.dot @@ -10,7 +10,7 @@ edge [fontname=helvetica] ; 2 -> 3 ; 4 [label="gini = 0.0\nsamples = 239\nvalue = [[239, 0]\n[0, 239]]", fillcolor="#e58139ff"] ; 3 -> 4 ; -5 [label="mean area <= 469.25\ngini = 0.444\nsamples = 3\nvalue = [[2, 1]\n[1, 2]]", fillcolor="#e5813913"] ; +5 [label="worst area <= 566.55\ngini = 0.444\nsamples = 3\nvalue = [[2, 1]\n[1, 2]]", fillcolor="#e5813913"] ; 3 -> 5 ; 6 [label="gini = 0.0\nsamples = 1\nvalue = [[0, 1]\n[1, 0]]", fillcolor="#e58139ff"] ; 5 -> 6 ; @@ -30,7 +30,7 @@ edge [fontname=helvetica] ; 11 -> 13 ; 14 [label="worst texture <= 20.645\ngini = 0.202\nsamples = 167\nvalue = [[19, 148]\n[148, 19]]", fillcolor="#e5813994"] ; 0 -> 14 [labeldistance=2.5, labelangle=-45, headlabel="False"] ; -15 [label="worst perimeter <= 116.8\ngini = 0.375\nsamples = 16\nvalue = [[12, 4]\n[4, 12]]", fillcolor="#e5813938"] ; +15 [label="worst radius <= 17.74\ngini = 0.375\nsamples = 16\nvalue = [[12, 4]\n[4, 12]]", fillcolor="#e5813938"] ; 14 -> 15 ; 16 [label="gini = 0.0\nsamples = 11\nvalue = [[11, 0]\n[0, 11]]", fillcolor="#e58139ff"] ; 15 -> 16 ; diff --git a/doc/pub/DecisionTrees/ipynb/DataFiles/cancer.png b/doc/pub/DecisionTrees/ipynb/DataFiles/cancer.png index 045c6e0a3..4626c9fe1 100644 Binary files a/doc/pub/DecisionTrees/ipynb/DataFiles/cancer.png and b/doc/pub/DecisionTrees/ipynb/DataFiles/cancer.png differ diff --git a/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb b/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb index a45625ff9..a27040afc 100644 --- a/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb +++ b/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb @@ -10,7 +10,7 @@ " \n", "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n", "\n", - "Date: **Oct 31, 2019**\n", + "Date: **Nov 1, 2019**\n", "\n", "Copyright 1999-2019, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license\n", "\n", @@ -1411,7 +1411,7 @@ "amount that the Gini index is decreased by splits over a given\n", "predictor, averaged over all $B$ trees.\n", "\n", - "## Simple example, head or tail" + "## Simple Voting Example, head or tail" ] }, { @@ -1440,7 +1440,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Bagging Example" + "## Using the Voting Classifier" ] }, { @@ -1496,6 +1496,178 @@ " print(clf.__class__.__name__, accuracy_score(y_test, y_pred))" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Please, not the moons again! Voting and Bagging" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "from sklearn.datasets import make_moons\n", + "\n", + "X, y = make_moons(n_samples=500, noise=0.30, random_state=42)\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.ensemble import VotingClassifier\n", + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.svm import SVC\n", + "\n", + "log_clf = LogisticRegression(random_state=42)\n", + "rnd_clf = RandomForestClassifier(random_state=42)\n", + "svm_clf = SVC(random_state=42)\n", + "\n", + "voting_clf = VotingClassifier(\n", + " estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n", + " voting='hard')\n", + "voting_clf.fit(X_train, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.metrics import accuracy_score\n", + "\n", + "for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n", + " clf.fit(X_train, y_train)\n", + " y_pred = clf.predict(X_test)\n", + " print(clf.__class__.__name__, accuracy_score(y_test, y_pred))" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "log_clf = LogisticRegression(random_state=42)\n", + "rnd_clf = RandomForestClassifier(random_state=42)\n", + "svm_clf = SVC(probability=True, random_state=42)\n", + "\n", + "voting_clf = VotingClassifier(\n", + " estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n", + " voting='soft')\n", + "voting_clf.fit(X_train, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.metrics import accuracy_score\n", + "\n", + "for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n", + " clf.fit(X_train, y_train)\n", + " y_pred = clf.predict(X_test)\n", + " print(clf.__class__.__name__, accuracy_score(y_test, y_pred))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Now Bagging" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.ensemble import BaggingClassifier\n", + "from sklearn.tree import DecisionTreeClassifier\n", + "\n", + "bag_clf = BaggingClassifier(\n", + " DecisionTreeClassifier(random_state=42), n_estimators=500,\n", + " max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)\n", + "bag_clf.fit(X_train, y_train)\n", + "y_pred = bag_clf.predict(X_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.metrics import accuracy_score\n", + "print(accuracy_score(y_test, y_pred))" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tree_clf = DecisionTreeClassifier(random_state=42)\n", + "tree_clf.fit(X_train, y_train)\n", + "y_pred_tree = tree_clf.predict(X_test)\n", + "print(accuracy_score(y_test, y_pred_tree))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from matplotlib.colors import ListedColormap\n", + "\n", + "def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):\n", + " x1s = np.linspace(axes[0], axes[1], 100)\n", + " x2s = np.linspace(axes[2], axes[3], 100)\n", + " x1, x2 = np.meshgrid(x1s, x2s)\n", + " X_new = np.c_[x1.ravel(), x2.ravel()]\n", + " y_pred = clf.predict(X_new).reshape(x1.shape)\n", + " custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])\n", + " plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)\n", + " if contour:\n", + " custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])\n", + " plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)\n", + " plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"yo\", alpha=alpha)\n", + " plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"bs\", alpha=alpha)\n", + " plt.axis(axes)\n", + " plt.xlabel(r\"$x_1$\", fontsize=18)\n", + " plt.ylabel(r\"$x_2$\", fontsize=18, rotation=0)\n", + "plt.figure(figsize=(11,4))\n", + "plt.subplot(121)\n", + "plot_decision_boundary(tree_clf, X, y)\n", + "plt.title(\"Decision Tree\", fontsize=14)\n", + "plt.subplot(122)\n", + "plot_decision_boundary(bag_clf, X, y)\n", + "plt.title(\"Decision Trees with Bagging\", fontsize=14)\n", + "plt.show()" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -1551,7 +1723,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 24, "metadata": { "collapsed": false }, @@ -1569,178 +1741,6 @@ "accuracy = cross_validate(Random_Forest_model,X,Y,cv=10)['test_score']" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Please, not the moons again!" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "from sklearn.model_selection import train_test_split\n", - "from sklearn.datasets import make_moons\n", - "\n", - "X, y = make_moons(n_samples=500, noise=0.30, random_state=42)\n", - "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)\n", - "from sklearn.ensemble import RandomForestClassifier\n", - "from sklearn.ensemble import VotingClassifier\n", - "from sklearn.linear_model import LogisticRegression\n", - "from sklearn.svm import SVC\n", - "\n", - "log_clf = LogisticRegression(random_state=42)\n", - "rnd_clf = RandomForestClassifier(random_state=42)\n", - "svm_clf = SVC(random_state=42)\n", - "\n", - "voting_clf = VotingClassifier(\n", - " estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n", - " voting='hard')\n", - "voting_clf.fit(X_train, y_train)" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "from sklearn.metrics import accuracy_score\n", - "\n", - "for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n", - " clf.fit(X_train, y_train)\n", - " y_pred = clf.predict(X_test)\n", - " print(clf.__class__.__name__, accuracy_score(y_test, y_pred))" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "log_clf = LogisticRegression(random_state=42)\n", - "rnd_clf = RandomForestClassifier(random_state=42)\n", - "svm_clf = SVC(probability=True, random_state=42)\n", - "\n", - "voting_clf = VotingClassifier(\n", - " estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n", - " voting='soft')\n", - "voting_clf.fit(X_train, y_train)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "from sklearn.metrics import accuracy_score\n", - "\n", - "for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n", - " clf.fit(X_train, y_train)\n", - " y_pred = clf.predict(X_test)\n", - " print(clf.__class__.__name__, accuracy_score(y_test, y_pred))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Bagging examples" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "from sklearn.ensemble import BaggingClassifier\n", - "from sklearn.tree import DecisionTreeClassifier\n", - "\n", - "bag_clf = BaggingClassifier(\n", - " DecisionTreeClassifier(random_state=42), n_estimators=500,\n", - " max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)\n", - "bag_clf.fit(X_train, y_train)\n", - "y_pred = bag_clf.predict(X_test)" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "from sklearn.metrics import accuracy_score\n", - "print(accuracy_score(y_test, y_pred))" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "tree_clf = DecisionTreeClassifier(random_state=42)\n", - "tree_clf.fit(X_train, y_train)\n", - "y_pred_tree = tree_clf.predict(X_test)\n", - "print(accuracy_score(y_test, y_pred_tree))" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "from matplotlib.colors import ListedColormap\n", - "\n", - "def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):\n", - " x1s = np.linspace(axes[0], axes[1], 100)\n", - " x2s = np.linspace(axes[2], axes[3], 100)\n", - " x1, x2 = np.meshgrid(x1s, x2s)\n", - " X_new = np.c_[x1.ravel(), x2.ravel()]\n", - " y_pred = clf.predict(X_new).reshape(x1.shape)\n", - " custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])\n", - " plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)\n", - " if contour:\n", - " custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])\n", - " plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)\n", - " plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"yo\", alpha=alpha)\n", - " plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"bs\", alpha=alpha)\n", - " plt.axis(axes)\n", - " plt.xlabel(r\"$x_1$\", fontsize=18)\n", - " plt.ylabel(r\"$x_2$\", fontsize=18, rotation=0)\n", - "plt.figure(figsize=(11,4))\n", - "plt.subplot(121)\n", - "plot_decision_boundary(tree_clf, X, y)\n", - "plt.title(\"Decision Tree\", fontsize=14)\n", - "plt.subplot(122)\n", - "plot_decision_boundary(bag_clf, X, y)\n", - "plt.title(\"Decision Trees with Bagging\", fontsize=14)\n", - "plt.show()" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -1777,6 +1777,194 @@ "y_pred_rf = rnd_clf.predict(X_test)\n", "np.sum(y_pred == y_pred_rf) / len(y_pred)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Feature Importance" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "try:\n", + " from sklearn.datasets import fetch_openml\n", + " mnist = fetch_openml('mnist_784', version=1)\n", + " mnist.target = mnist.target.astype(np.int64)\n", + "except ImportError:\n", + " from sklearn.datasets import fetch_mldata\n", + " mnist = fetch_mldata('MNIST original')\n", + "\n", + "rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)\n", + "rnd_clf.fit(mnist[\"data\"], mnist[\"target\"])\n", + "\n", + "def plot_digit(data):\n", + " image = data.reshape(28, 28)\n", + " plt.imshow(image, cmap = mpl.cm.hot,\n", + " interpolation=\"nearest\")\n", + " plt.axis(\"off\")\n", + "\n", + "plot_digit(rnd_clf.feature_importances_)\n", + "\n", + "cbar = plt.colorbar(ticks=[rnd_clf.feature_importances_.min(), rnd_clf.feature_importances_.max()])\n", + "cbar.ax.set_yticklabels(['Not important', 'Very important'])\n", + "\n", + "#save_fig(\"mnist_feature_importance_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Boosting: AdaBoost" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.ensemble import AdaBoostClassifier\n", + "\n", + "ada_clf = AdaBoostClassifier(\n", + " DecisionTreeClassifier(max_depth=1), n_estimators=200,\n", + " algorithm=\"SAMME.R\", learning_rate=0.5, random_state=42)\n", + "ada_clf.fit(X_train, y_train)\n", + "\n", + "plot_decision_boundary(ada_clf, X, y)\n", + "\n", + "m = len(X_train)\n", + "\n", + "plt.figure(figsize=(11, 4))\n", + "for subplot, learning_rate in ((121, 1), (122, 0.5)):\n", + " sample_weights = np.ones(m)\n", + " plt.subplot(subplot)\n", + " for i in range(5):\n", + " svm_clf = SVC(kernel=\"rbf\", C=0.05, gamma=\"auto\", random_state=42)\n", + " svm_clf.fit(X_train, y_train, sample_weight=sample_weights)\n", + " y_pred = svm_clf.predict(X_train)\n", + " sample_weights[y_pred != y_train] *= (1 + learning_rate)\n", + " plot_decision_boundary(svm_clf, X, y, alpha=0.2)\n", + " plt.title(\"learning_rate = {}\".format(learning_rate), fontsize=16)\n", + " if subplot == 121:\n", + " plt.text(-0.7, -0.65, \"1\", fontsize=14)\n", + " plt.text(-0.6, -0.10, \"2\", fontsize=14)\n", + " plt.text(-0.5, 0.10, \"3\", fontsize=14)\n", + " plt.text(-0.4, 0.55, \"4\", fontsize=14)\n", + " plt.text(-0.3, 0.90, \"5\", fontsize=14)\n", + "\n", + "save_fig(\"boosting_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Gradient Boosting" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "X = np.random.rand(100, 1) - 0.5\n", + "y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)\n", + "\n", + "from sklearn.tree import DecisionTreeRegressor\n", + "\n", + "tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)\n", + "tree_reg1.fit(X, y)\n", + "\n", + "y2 = y - tree_reg1.predict(X)\n", + "tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)\n", + "tree_reg2.fit(X, y2)\n", + "\n", + "y3 = y2 - tree_reg2.predict(X)\n", + "tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)\n", + "tree_reg3.fit(X, y3)\n", + "\n", + "X_new = np.array([[0.8]])\n", + "y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))\n", + "\n", + "def plot_predictions(regressors, X, y, axes, label=None, style=\"r-\", data_style=\"b.\", data_label=None):\n", + " x1 = np.linspace(axes[0], axes[1], 500)\n", + " y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)\n", + " plt.plot(X[:, 0], y, data_style, label=data_label)\n", + " plt.plot(x1, y_pred, style, linewidth=2, label=label)\n", + " if label or data_label:\n", + " plt.legend(loc=\"upper center\", fontsize=16)\n", + " plt.axis(axes)\n", + "\n", + "plt.figure(figsize=(11,11))\n", + "\n", + "plt.subplot(321)\n", + "plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h_1(x_1)$\", style=\"g-\", data_label=\"Training set\")\n", + "plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n", + "plt.title(\"Residuals and tree predictions\", fontsize=16)\n", + "\n", + "plt.subplot(322)\n", + "plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h(x_1) = h_1(x_1)$\", data_label=\"Training set\")\n", + "plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n", + "plt.title(\"Ensemble predictions\", fontsize=16)\n", + "\n", + "plt.subplot(323)\n", + "plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label=\"$h_2(x_1)$\", style=\"g-\", data_style=\"k+\", data_label=\"Residuals\")\n", + "plt.ylabel(\"$y - h_1(x_1)$\", fontsize=16)\n", + "\n", + "plt.subplot(324)\n", + "plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h(x_1) = h_1(x_1) + h_2(x_1)$\")\n", + "plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n", + "\n", + "plt.subplot(325)\n", + "plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label=\"$h_3(x_1)$\", style=\"g-\", data_style=\"k+\")\n", + "plt.ylabel(\"$y - h_1(x_1) - h_2(x_1)$\", fontsize=16)\n", + "plt.xlabel(\"$x_1$\", fontsize=16)\n", + "\n", + "plt.subplot(326)\n", + "plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$\")\n", + "plt.xlabel(\"$x_1$\", fontsize=16)\n", + "plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n", + "\n", + "save_fig(\"gradient_boosting_plot\")\n", + "plt.show()\n", + "\n", + "from sklearn.ensemble import GradientBoostingRegressor\n", + "\n", + "gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0, random_state=42)\n", + "gbrt.fit(X, y)\n", + "\n", + "gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)\n", + "gbrt_slow.fit(X, y)\n", + "\n", + "plt.figure(figsize=(11,4))\n", + "\n", + "plt.subplot(121)\n", + "plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"Ensemble predictions\")\n", + "plt.title(\"learning_rate={}, n_estimators={}\".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)\n", + "\n", + "plt.subplot(122)\n", + "plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])\n", + "plt.title(\"learning_rate={}, n_estimators={}\".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)\n", + "\n", + "save_fig(\"gbrt_learning_rate_plot\")\n", + "plt.show()" + ] } ], "metadata": {}, diff --git a/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz b/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz index 52ccca548..63c74dc20 100644 Binary files a/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz and b/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz differ diff --git a/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf b/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf index a5b0a8c31..9cd162707 100644 Binary files a/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf and b/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf differ diff --git a/doc/src/DecisionTrees/DecisionTrees.do.txt b/doc/src/DecisionTrees/DecisionTrees.do.txt index d183fceeb..6a3a32a28 100644 --- a/doc/src/DecisionTrees/DecisionTrees.do.txt +++ b/doc/src/DecisionTrees/DecisionTrees.do.txt @@ -1169,7 +1169,7 @@ amount that the Gini index is decreased by splits over a given predictor, averaged over all $B$ trees. !split -===== Simple example, head or tail ===== +===== Simple Voting Example, head or tail ===== !bc pycod heads_proba = 0.51 coin_tosses = (np.random.rand(10000, 10) < heads_proba).astype(np.int32) @@ -1187,7 +1187,7 @@ plt.show() !ec !split -===== Bagging Example ===== +===== Using the Voting Classifier ===== !bc pycod from sklearn.model_selection import train_test_split from sklearn.datasets import make_moons @@ -1235,6 +1235,113 @@ for clf in (log_clf, rnd_clf, svm_clf, voting_clf): !ec +!split +===== Please, not the moons again! Voting and Bagging ===== +!bc pycod +from sklearn.model_selection import train_test_split +from sklearn.datasets import make_moons + +X, y = make_moons(n_samples=500, noise=0.30, random_state=42) +X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) +from sklearn.ensemble import RandomForestClassifier +from sklearn.ensemble import VotingClassifier +from sklearn.linear_model import LogisticRegression +from sklearn.svm import SVC + +log_clf = LogisticRegression(random_state=42) +rnd_clf = RandomForestClassifier(random_state=42) +svm_clf = SVC(random_state=42) + +voting_clf = VotingClassifier( + estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)], + voting='hard') +voting_clf.fit(X_train, y_train) +!ec + +!bc pycod +from sklearn.metrics import accuracy_score + +for clf in (log_clf, rnd_clf, svm_clf, voting_clf): + clf.fit(X_train, y_train) + y_pred = clf.predict(X_test) + print(clf.__class__.__name__, accuracy_score(y_test, y_pred)) +!ec + +!bc pycod +log_clf = LogisticRegression(random_state=42) +rnd_clf = RandomForestClassifier(random_state=42) +svm_clf = SVC(probability=True, random_state=42) + +voting_clf = VotingClassifier( + estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)], + voting='soft') +voting_clf.fit(X_train, y_train) +!ec + +!bc pycod +from sklearn.metrics import accuracy_score + +for clf in (log_clf, rnd_clf, svm_clf, voting_clf): + clf.fit(X_train, y_train) + y_pred = clf.predict(X_test) + print(clf.__class__.__name__, accuracy_score(y_test, y_pred)) +!ec + +!split +===== Now Bagging ===== + +!bc pycod +from sklearn.ensemble import BaggingClassifier +from sklearn.tree import DecisionTreeClassifier + +bag_clf = BaggingClassifier( + DecisionTreeClassifier(random_state=42), n_estimators=500, + max_samples=100, bootstrap=True, n_jobs=-1, random_state=42) +bag_clf.fit(X_train, y_train) +y_pred = bag_clf.predict(X_test) +!ec + + +!bc pycod +from sklearn.metrics import accuracy_score +print(accuracy_score(y_test, y_pred)) +!ec + +!bc pycod +tree_clf = DecisionTreeClassifier(random_state=42) +tree_clf.fit(X_train, y_train) +y_pred_tree = tree_clf.predict(X_test) +print(accuracy_score(y_test, y_pred_tree)) +!ec + +!bc pycod +from matplotlib.colors import ListedColormap + +def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True): + x1s = np.linspace(axes[0], axes[1], 100) + x2s = np.linspace(axes[2], axes[3], 100) + x1, x2 = np.meshgrid(x1s, x2s) + X_new = np.c_[x1.ravel(), x2.ravel()] + y_pred = clf.predict(X_new).reshape(x1.shape) + custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0']) + plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap) + if contour: + custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50']) + plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8) + plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha) + plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha) + plt.axis(axes) + plt.xlabel(r"$x_1$", fontsize=18) + plt.ylabel(r"$x_2$", fontsize=18, rotation=0) +plt.figure(figsize=(11,4)) +plt.subplot(121) +plot_decision_boundary(tree_clf, X, y) +plt.title("Decision Tree", fontsize=14) +plt.subplot(122) +plot_decision_boundary(bag_clf, X, y) +plt.title("Decision Trees with Bagging", fontsize=14) +plt.show() +!ec !split @@ -1291,113 +1398,6 @@ Random_Forest_model = RandomForestClassifier(n_estimators=100,criterion="entropy accuracy = cross_validate(Random_Forest_model,X,Y,cv=10)['test_score'] !ec -!split -===== Please, not the moons again! ===== -!bc pycod -from sklearn.model_selection import train_test_split -from sklearn.datasets import make_moons - -X, y = make_moons(n_samples=500, noise=0.30, random_state=42) -X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) -from sklearn.ensemble import RandomForestClassifier -from sklearn.ensemble import VotingClassifier -from sklearn.linear_model import LogisticRegression -from sklearn.svm import SVC - -log_clf = LogisticRegression(random_state=42) -rnd_clf = RandomForestClassifier(random_state=42) -svm_clf = SVC(random_state=42) - -voting_clf = VotingClassifier( - estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)], - voting='hard') -voting_clf.fit(X_train, y_train) -!ec - -!bc pycod -from sklearn.metrics import accuracy_score - -for clf in (log_clf, rnd_clf, svm_clf, voting_clf): - clf.fit(X_train, y_train) - y_pred = clf.predict(X_test) - print(clf.__class__.__name__, accuracy_score(y_test, y_pred)) -!ec - -!bc pycod -log_clf = LogisticRegression(random_state=42) -rnd_clf = RandomForestClassifier(random_state=42) -svm_clf = SVC(probability=True, random_state=42) - -voting_clf = VotingClassifier( - estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)], - voting='soft') -voting_clf.fit(X_train, y_train) -!ec - -!bc pycod -from sklearn.metrics import accuracy_score - -for clf in (log_clf, rnd_clf, svm_clf, voting_clf): - clf.fit(X_train, y_train) - y_pred = clf.predict(X_test) - print(clf.__class__.__name__, accuracy_score(y_test, y_pred)) -!ec - -!split -===== Bagging examples ===== - -!bc pycod -from sklearn.ensemble import BaggingClassifier -from sklearn.tree import DecisionTreeClassifier - -bag_clf = BaggingClassifier( - DecisionTreeClassifier(random_state=42), n_estimators=500, - max_samples=100, bootstrap=True, n_jobs=-1, random_state=42) -bag_clf.fit(X_train, y_train) -y_pred = bag_clf.predict(X_test) -!ec - - -!bc pycod -from sklearn.metrics import accuracy_score -print(accuracy_score(y_test, y_pred)) -!ec - -!bc pycod -tree_clf = DecisionTreeClassifier(random_state=42) -tree_clf.fit(X_train, y_train) -y_pred_tree = tree_clf.predict(X_test) -print(accuracy_score(y_test, y_pred_tree)) -!ec - -!bc pycod -from matplotlib.colors import ListedColormap - -def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True): - x1s = np.linspace(axes[0], axes[1], 100) - x2s = np.linspace(axes[2], axes[3], 100) - x1, x2 = np.meshgrid(x1s, x2s) - X_new = np.c_[x1.ravel(), x2.ravel()] - y_pred = clf.predict(X_new).reshape(x1.shape) - custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0']) - plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap) - if contour: - custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50']) - plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8) - plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha) - plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha) - plt.axis(axes) - plt.xlabel(r"$x_1$", fontsize=18) - plt.ylabel(r"$x_2$", fontsize=18, rotation=0) -plt.figure(figsize=(11,4)) -plt.subplot(121) -plot_decision_boundary(tree_clf, X, y) -plt.title("Decision Tree", fontsize=14) -plt.subplot(122) -plot_decision_boundary(bag_clf, X, y) -plt.title("Decision Trees with Bagging", fontsize=14) -plt.show() -!ec !split ===== Then random forests ===== @@ -1421,3 +1421,161 @@ np.sum(y_pred == y_pred_rf) / len(y_pred) +!split +===== Feature Importance ===== + +!bc pycod +try: + from sklearn.datasets import fetch_openml + mnist = fetch_openml('mnist_784', version=1) + mnist.target = mnist.target.astype(np.int64) +except ImportError: + from sklearn.datasets import fetch_mldata + mnist = fetch_mldata('MNIST original') + +rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42) +rnd_clf.fit(mnist["data"], mnist["target"]) + +def plot_digit(data): + image = data.reshape(28, 28) + plt.imshow(image, cmap = mpl.cm.hot, + interpolation="nearest") + plt.axis("off") + +plot_digit(rnd_clf.feature_importances_) + +cbar = plt.colorbar(ticks=[rnd_clf.feature_importances_.min(), rnd_clf.feature_importances_.max()]) +cbar.ax.set_yticklabels(['Not important', 'Very important']) + +#save_fig("mnist_feature_importance_plot") +plt.show() +!ec + +!split +===== Boosting: AdaBoost ===== + +!bc pycod +from sklearn.ensemble import AdaBoostClassifier + +ada_clf = AdaBoostClassifier( + DecisionTreeClassifier(max_depth=1), n_estimators=200, + algorithm="SAMME.R", learning_rate=0.5, random_state=42) +ada_clf.fit(X_train, y_train) + +plot_decision_boundary(ada_clf, X, y) + +m = len(X_train) + +plt.figure(figsize=(11, 4)) +for subplot, learning_rate in ((121, 1), (122, 0.5)): + sample_weights = np.ones(m) + plt.subplot(subplot) + for i in range(5): + svm_clf = SVC(kernel="rbf", C=0.05, gamma="auto", random_state=42) + svm_clf.fit(X_train, y_train, sample_weight=sample_weights) + y_pred = svm_clf.predict(X_train) + sample_weights[y_pred != y_train] *= (1 + learning_rate) + plot_decision_boundary(svm_clf, X, y, alpha=0.2) + plt.title("learning_rate = {}".format(learning_rate), fontsize=16) + if subplot == 121: + plt.text(-0.7, -0.65, "1", fontsize=14) + plt.text(-0.6, -0.10, "2", fontsize=14) + plt.text(-0.5, 0.10, "3", fontsize=14) + plt.text(-0.4, 0.55, "4", fontsize=14) + plt.text(-0.3, 0.90, "5", fontsize=14) + +save_fig("boosting_plot") +plt.show() + + + +!ec + + +!split +===== Gradient Boosting ===== +!bc pycod +np.random.seed(42) +X = np.random.rand(100, 1) - 0.5 +y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100) + +from sklearn.tree import DecisionTreeRegressor + +tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42) +tree_reg1.fit(X, y) + +y2 = y - tree_reg1.predict(X) +tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42) +tree_reg2.fit(X, y2) + +y3 = y2 - tree_reg2.predict(X) +tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42) +tree_reg3.fit(X, y3) + +X_new = np.array([[0.8]]) +y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3)) + +def plot_predictions(regressors, X, y, axes, label=None, style="r-", data_style="b.", data_label=None): + x1 = np.linspace(axes[0], axes[1], 500) + y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors) + plt.plot(X[:, 0], y, data_style, label=data_label) + plt.plot(x1, y_pred, style, linewidth=2, label=label) + if label or data_label: + plt.legend(loc="upper center", fontsize=16) + plt.axis(axes) + +plt.figure(figsize=(11,11)) + +plt.subplot(321) +plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h_1(x_1)$", style="g-", data_label="Training set") +plt.ylabel("$y$", fontsize=16, rotation=0) +plt.title("Residuals and tree predictions", fontsize=16) + +plt.subplot(322) +plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1)$", data_label="Training set") +plt.ylabel("$y$", fontsize=16, rotation=0) +plt.title("Ensemble predictions", fontsize=16) + +plt.subplot(323) +plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_2(x_1)$", style="g-", data_style="k+", data_label="Residuals") +plt.ylabel("$y - h_1(x_1)$", fontsize=16) + +plt.subplot(324) +plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1)$") +plt.ylabel("$y$", fontsize=16, rotation=0) + +plt.subplot(325) +plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_3(x_1)$", style="g-", data_style="k+") +plt.ylabel("$y - h_1(x_1) - h_2(x_1)$", fontsize=16) +plt.xlabel("$x_1$", fontsize=16) + +plt.subplot(326) +plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$") +plt.xlabel("$x_1$", fontsize=16) +plt.ylabel("$y$", fontsize=16, rotation=0) + +save_fig("gradient_boosting_plot") +plt.show() + +from sklearn.ensemble import GradientBoostingRegressor + +gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0, random_state=42) +gbrt.fit(X, y) + +gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42) +gbrt_slow.fit(X, y) + +plt.figure(figsize=(11,4)) + +plt.subplot(121) +plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="Ensemble predictions") +plt.title("learning_rate={}, n_estimators={}".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14) + +plt.subplot(122) +plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8]) +plt.title("learning_rate={}, n_estimators={}".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14) + +save_fig("gbrt_learning_rate_plot") +plt.show() + +!ec