+ + + + +
+ + +
heads_proba = 0.51
+coin_tosses = (np.random.rand(10000, 10) < heads_proba).astype(np.int32)
+cumulative_heads_ratio = np.cumsum(coin_tosses, axis=0) / np.arange(1, 10001).reshape(-1, 1)
+plt.figure(figsize=(8,3.5))
+plt.plot(cumulative_heads_ratio)
+plt.plot([0, 10000], [0.51, 0.51], "k--", linewidth=2, label="51%")
+plt.plot([0, 10000], [0.5, 0.5], "k-", label="50%")
+plt.xlabel("Number of coin tosses")
+plt.ylabel("Heads ratio")
+plt.legend(loc="lower right")
+plt.axis([0, 10000, 0.42, 0.58])
+plt.show()
++
+ +
+ + +
+ + + + +
+ + +
from sklearn.model_selection import train_test_split
+from sklearn.datasets import make_moons
+
+X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
+X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
+
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.ensemble import VotingClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.svm import SVC
+
+log_clf = LogisticRegression(solver="liblinear", random_state=42)
+rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
+svm_clf = SVC(gamma="auto", random_state=42)
+
+voting_clf = VotingClassifier(
+ estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
+ voting='hard')
+
+voting_clf.fit(X_train, y_train)
+
+from sklearn.metrics import accuracy_score
+
+for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
+ clf.fit(X_train, y_train)
+ y_pred = clf.predict(X_test)
+ print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
+
+log_clf = LogisticRegression(solver="liblinear", random_state=42)
+rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
+svm_clf = SVC(gamma="auto", probability=True, random_state=42)
+
+voting_clf = VotingClassifier(
+ estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
+ voting='soft')
+voting_clf.fit(X_train, y_train)
+
+from sklearn.metrics import accuracy_score
+
+for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
+ clf.fit(X_train, y_train)
+ y_pred = clf.predict(X_test)
+ print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
++
+ +
+ + +
+ + + + +
+Random forests provide an improvement over bagged trees by way of a +small tweak that decorrelates the trees. + +
+As in bagging, we build a +number of decision trees on bootstrapped training samples. But when +building these decision trees, each time a split in a tree is +considered, a random sample of \( m \) predictors is chosen as split +candidates from the full set of \( p \) predictors. The split is allowed to +use only one of those \( m \) predictors. + +
+A fresh sample of \( m \) predictors is +taken at each split, and typically we choose + +$$ +m\approx \sqrt{p}. +$$ + +
+In building a random forest, at +each split in the tree, the algorithm is not even allowed to consider +a majority of the available predictors. + +
+The reason for this is rather clever. Suppose that there is one very +strong predictor in the data set, along with a number of other +moderately strong predictors. Then in the collection of bagged +variable importance random forest trees, most or all of the trees will +use this strong predictor in the top split. Consequently, all of the +bagged trees will look quite similar to each other. Hence the +predictions from the bagged trees will be highly correlated. +Unfortunately, averaging many highly correlated quantities does not +lead to as large of a reduction in variance as averaging many +uncorrelated quanti- ties. In particular, this means that bagging will +not lead to a substantial reduction in variance over a single tree in +this setting. + +
+
+ +
+ + +
+ + + + +
+ + +
from sklearn.ensemble import RandomForestClassifier
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+# Data set not specificied
+X = dataset.XXX
+Y = dataset.YYY
+#Instantiate the model with 100 trees and entropy as splitting criteria
+Random_Forest_model = RandomForestClassifier(n_estimators=100,criterion="entropy")
+#Cross validation
+accuracy = cross_validate(Random_Forest_model,X,Y,cv=10)['test_score']
++
+ +
+ + +
+ + + + +
+ + +
from sklearn.model_selection import train_test_split
+from sklearn.datasets import make_moons
+
+X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
+X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.ensemble import VotingClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.svm import SVC
+
+log_clf = LogisticRegression(random_state=42)
+rnd_clf = RandomForestClassifier(random_state=42)
+svm_clf = SVC(random_state=42)
+
+voting_clf = VotingClassifier(
+ estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
+ voting='hard')
+voting_clf.fit(X_train, y_train)
++ + +
from sklearn.metrics import accuracy_score
+
+for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
+ clf.fit(X_train, y_train)
+ y_pred = clf.predict(X_test)
+ print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
++ + +
log_clf = LogisticRegression(random_state=42)
+rnd_clf = RandomForestClassifier(random_state=42)
+svm_clf = SVC(probability=True, random_state=42)
+
+voting_clf = VotingClassifier(
+ estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
+ voting='soft')
+voting_clf.fit(X_train, y_train)
++ + +
from sklearn.metrics import accuracy_score
+
+for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
+ clf.fit(X_train, y_train)
+ y_pred = clf.predict(X_test)
+ print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
++
+ +
+ + +
+ + + + +
+ + +
from sklearn.ensemble import BaggingClassifier
+from sklearn.tree import DecisionTreeClassifier
+
+bag_clf = BaggingClassifier(
+ DecisionTreeClassifier(random_state=42), n_estimators=500,
+ max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)
+bag_clf.fit(X_train, y_train)
+y_pred = bag_clf.predict(X_test)
++ + +
from sklearn.metrics import accuracy_score
+print(accuracy_score(y_test, y_pred))
++ + +
tree_clf = DecisionTreeClassifier(random_state=42)
+tree_clf.fit(X_train, y_train)
+y_pred_tree = tree_clf.predict(X_test)
+print(accuracy_score(y_test, y_pred_tree))
++ + +
from matplotlib.colors import ListedColormap
+
+def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):
+ x1s = np.linspace(axes[0], axes[1], 100)
+ x2s = np.linspace(axes[2], axes[3], 100)
+ x1, x2 = np.meshgrid(x1s, x2s)
+ X_new = np.c_[x1.ravel(), x2.ravel()]
+ y_pred = clf.predict(X_new).reshape(x1.shape)
+ custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
+ plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
+ if contour:
+ custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
+ plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
+ plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha)
+ plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha)
+ plt.axis(axes)
+ plt.xlabel(r"$x_1$", fontsize=18)
+ plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
+plt.figure(figsize=(11,4))
+plt.subplot(121)
+plot_decision_boundary(tree_clf, X, y)
+plt.title("Decision Tree", fontsize=14)
+plt.subplot(122)
+plot_decision_boundary(bag_clf, X, y)
+plt.title("Decision Trees with Bagging", fontsize=14)
+plt.show()
++
+ +
+ + +
+ + + + +
+ + +
bag_clf = BaggingClassifier(
+ DecisionTreeClassifier(splitter="random", max_leaf_nodes=16, random_state=42),
+ n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1, random_state=42)
++ + +
bag_clf.fit(X_train, y_train)
+y_pred = bag_clf.predict(X_test)
+from sklearn.ensemble import RandomForestClassifier
+rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)
+rnd_clf.fit(X_train, y_train)
+y_pred_rf = rnd_clf.predict(X_test)
+np.sum(y_pred == y_pred_rf) / len(y_pred)
++ +
+ +
+ + +