update on decision trees
This commit is contained in:
mhjensen
@@ -1169,7 +1169,7 @@ amount that the Gini index is decreased by splits over a given
|
||||
predictor, averaged over all $B$ trees.
|
||||
|
||||
!split
|
||||
===== Simple example, head or tail =====
|
||||
===== Simple Voting Example, head or tail =====
|
||||
!bc pycod
|
||||
heads_proba = 0.51
|
||||
coin_tosses = (np.random.rand(10000, 10) < heads_proba).astype(np.int32)
|
||||
@@ -1187,7 +1187,7 @@ plt.show()
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Bagging Example =====
|
||||
===== Using the Voting Classifier =====
|
||||
!bc pycod
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.datasets import make_moons
|
||||
@@ -1235,6 +1235,113 @@ for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Please, not the moons again! Voting and Bagging =====
|
||||
!bc pycod
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.datasets import make_moons
|
||||
|
||||
X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
|
||||
from sklearn.ensemble import RandomForestClassifier
|
||||
from sklearn.ensemble import VotingClassifier
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.svm import SVC
|
||||
|
||||
log_clf = LogisticRegression(random_state=42)
|
||||
rnd_clf = RandomForestClassifier(random_state=42)
|
||||
svm_clf = SVC(random_state=42)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
|
||||
voting='hard')
|
||||
voting_clf.fit(X_train, y_train)
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
from sklearn.metrics import accuracy_score
|
||||
|
||||
for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
log_clf = LogisticRegression(random_state=42)
|
||||
rnd_clf = RandomForestClassifier(random_state=42)
|
||||
svm_clf = SVC(probability=True, random_state=42)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
|
||||
voting='soft')
|
||||
voting_clf.fit(X_train, y_train)
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
from sklearn.metrics import accuracy_score
|
||||
|
||||
for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Now Bagging =====
|
||||
|
||||
!bc pycod
|
||||
from sklearn.ensemble import BaggingClassifier
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
|
||||
bag_clf = BaggingClassifier(
|
||||
DecisionTreeClassifier(random_state=42), n_estimators=500,
|
||||
max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)
|
||||
bag_clf.fit(X_train, y_train)
|
||||
y_pred = bag_clf.predict(X_test)
|
||||
!ec
|
||||
|
||||
|
||||
!bc pycod
|
||||
from sklearn.metrics import accuracy_score
|
||||
print(accuracy_score(y_test, y_pred))
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
tree_clf = DecisionTreeClassifier(random_state=42)
|
||||
tree_clf.fit(X_train, y_train)
|
||||
y_pred_tree = tree_clf.predict(X_test)
|
||||
print(accuracy_score(y_test, y_pred_tree))
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
from matplotlib.colors import ListedColormap
|
||||
|
||||
def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):
|
||||
x1s = np.linspace(axes[0], axes[1], 100)
|
||||
x2s = np.linspace(axes[2], axes[3], 100)
|
||||
x1, x2 = np.meshgrid(x1s, x2s)
|
||||
X_new = np.c_[x1.ravel(), x2.ravel()]
|
||||
y_pred = clf.predict(X_new).reshape(x1.shape)
|
||||
custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
|
||||
plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
|
||||
if contour:
|
||||
custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
|
||||
plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
|
||||
plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha)
|
||||
plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha)
|
||||
plt.axis(axes)
|
||||
plt.xlabel(r"$x_1$", fontsize=18)
|
||||
plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
|
||||
plt.figure(figsize=(11,4))
|
||||
plt.subplot(121)
|
||||
plot_decision_boundary(tree_clf, X, y)
|
||||
plt.title("Decision Tree", fontsize=14)
|
||||
plt.subplot(122)
|
||||
plot_decision_boundary(bag_clf, X, y)
|
||||
plt.title("Decision Trees with Bagging", fontsize=14)
|
||||
plt.show()
|
||||
!ec
|
||||
|
||||
|
||||
!split
|
||||
@@ -1291,113 +1398,6 @@ Random_Forest_model = RandomForestClassifier(n_estimators=100,criterion="entropy
|
||||
accuracy = cross_validate(Random_Forest_model,X,Y,cv=10)['test_score']
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Please, not the moons again! =====
|
||||
!bc pycod
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.datasets import make_moons
|
||||
|
||||
X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
|
||||
from sklearn.ensemble import RandomForestClassifier
|
||||
from sklearn.ensemble import VotingClassifier
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.svm import SVC
|
||||
|
||||
log_clf = LogisticRegression(random_state=42)
|
||||
rnd_clf = RandomForestClassifier(random_state=42)
|
||||
svm_clf = SVC(random_state=42)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
|
||||
voting='hard')
|
||||
voting_clf.fit(X_train, y_train)
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
from sklearn.metrics import accuracy_score
|
||||
|
||||
for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
log_clf = LogisticRegression(random_state=42)
|
||||
rnd_clf = RandomForestClassifier(random_state=42)
|
||||
svm_clf = SVC(probability=True, random_state=42)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
|
||||
voting='soft')
|
||||
voting_clf.fit(X_train, y_train)
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
from sklearn.metrics import accuracy_score
|
||||
|
||||
for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Bagging examples =====
|
||||
|
||||
!bc pycod
|
||||
from sklearn.ensemble import BaggingClassifier
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
|
||||
bag_clf = BaggingClassifier(
|
||||
DecisionTreeClassifier(random_state=42), n_estimators=500,
|
||||
max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)
|
||||
bag_clf.fit(X_train, y_train)
|
||||
y_pred = bag_clf.predict(X_test)
|
||||
!ec
|
||||
|
||||
|
||||
!bc pycod
|
||||
from sklearn.metrics import accuracy_score
|
||||
print(accuracy_score(y_test, y_pred))
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
tree_clf = DecisionTreeClassifier(random_state=42)
|
||||
tree_clf.fit(X_train, y_train)
|
||||
y_pred_tree = tree_clf.predict(X_test)
|
||||
print(accuracy_score(y_test, y_pred_tree))
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
from matplotlib.colors import ListedColormap
|
||||
|
||||
def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):
|
||||
x1s = np.linspace(axes[0], axes[1], 100)
|
||||
x2s = np.linspace(axes[2], axes[3], 100)
|
||||
x1, x2 = np.meshgrid(x1s, x2s)
|
||||
X_new = np.c_[x1.ravel(), x2.ravel()]
|
||||
y_pred = clf.predict(X_new).reshape(x1.shape)
|
||||
custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
|
||||
plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
|
||||
if contour:
|
||||
custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
|
||||
plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
|
||||
plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha)
|
||||
plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha)
|
||||
plt.axis(axes)
|
||||
plt.xlabel(r"$x_1$", fontsize=18)
|
||||
plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
|
||||
plt.figure(figsize=(11,4))
|
||||
plt.subplot(121)
|
||||
plot_decision_boundary(tree_clf, X, y)
|
||||
plt.title("Decision Tree", fontsize=14)
|
||||
plt.subplot(122)
|
||||
plot_decision_boundary(bag_clf, X, y)
|
||||
plt.title("Decision Trees with Bagging", fontsize=14)
|
||||
plt.show()
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Then random forests =====
|
||||
@@ -1421,3 +1421,161 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== Feature Importance =====
|
||||
|
||||
!bc pycod
|
||||
try:
|
||||
from sklearn.datasets import fetch_openml
|
||||
mnist = fetch_openml('mnist_784', version=1)
|
||||
mnist.target = mnist.target.astype(np.int64)
|
||||
except ImportError:
|
||||
from sklearn.datasets import fetch_mldata
|
||||
mnist = fetch_mldata('MNIST original')
|
||||
|
||||
rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
|
||||
rnd_clf.fit(mnist["data"], mnist["target"])
|
||||
|
||||
def plot_digit(data):
|
||||
image = data.reshape(28, 28)
|
||||
plt.imshow(image, cmap = mpl.cm.hot,
|
||||
interpolation="nearest")
|
||||
plt.axis("off")
|
||||
|
||||
plot_digit(rnd_clf.feature_importances_)
|
||||
|
||||
cbar = plt.colorbar(ticks=[rnd_clf.feature_importances_.min(), rnd_clf.feature_importances_.max()])
|
||||
cbar.ax.set_yticklabels(['Not important', 'Very important'])
|
||||
|
||||
#save_fig("mnist_feature_importance_plot")
|
||||
plt.show()
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Boosting: AdaBoost =====
|
||||
|
||||
!bc pycod
|
||||
from sklearn.ensemble import AdaBoostClassifier
|
||||
|
||||
ada_clf = AdaBoostClassifier(
|
||||
DecisionTreeClassifier(max_depth=1), n_estimators=200,
|
||||
algorithm="SAMME.R", learning_rate=0.5, random_state=42)
|
||||
ada_clf.fit(X_train, y_train)
|
||||
|
||||
plot_decision_boundary(ada_clf, X, y)
|
||||
|
||||
m = len(X_train)
|
||||
|
||||
plt.figure(figsize=(11, 4))
|
||||
for subplot, learning_rate in ((121, 1), (122, 0.5)):
|
||||
sample_weights = np.ones(m)
|
||||
plt.subplot(subplot)
|
||||
for i in range(5):
|
||||
svm_clf = SVC(kernel="rbf", C=0.05, gamma="auto", random_state=42)
|
||||
svm_clf.fit(X_train, y_train, sample_weight=sample_weights)
|
||||
y_pred = svm_clf.predict(X_train)
|
||||
sample_weights[y_pred != y_train] *= (1 + learning_rate)
|
||||
plot_decision_boundary(svm_clf, X, y, alpha=0.2)
|
||||
plt.title("learning_rate = {}".format(learning_rate), fontsize=16)
|
||||
if subplot == 121:
|
||||
plt.text(-0.7, -0.65, "1", fontsize=14)
|
||||
plt.text(-0.6, -0.10, "2", fontsize=14)
|
||||
plt.text(-0.5, 0.10, "3", fontsize=14)
|
||||
plt.text(-0.4, 0.55, "4", fontsize=14)
|
||||
plt.text(-0.3, 0.90, "5", fontsize=14)
|
||||
|
||||
save_fig("boosting_plot")
|
||||
plt.show()
|
||||
|
||||
|
||||
|
||||
!ec
|
||||
|
||||
|
||||
!split
|
||||
===== Gradient Boosting =====
|
||||
!bc pycod
|
||||
np.random.seed(42)
|
||||
X = np.random.rand(100, 1) - 0.5
|
||||
y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)
|
||||
|
||||
from sklearn.tree import DecisionTreeRegressor
|
||||
|
||||
tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)
|
||||
tree_reg1.fit(X, y)
|
||||
|
||||
y2 = y - tree_reg1.predict(X)
|
||||
tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)
|
||||
tree_reg2.fit(X, y2)
|
||||
|
||||
y3 = y2 - tree_reg2.predict(X)
|
||||
tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)
|
||||
tree_reg3.fit(X, y3)
|
||||
|
||||
X_new = np.array([[0.8]])
|
||||
y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))
|
||||
|
||||
def plot_predictions(regressors, X, y, axes, label=None, style="r-", data_style="b.", data_label=None):
|
||||
x1 = np.linspace(axes[0], axes[1], 500)
|
||||
y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)
|
||||
plt.plot(X[:, 0], y, data_style, label=data_label)
|
||||
plt.plot(x1, y_pred, style, linewidth=2, label=label)
|
||||
if label or data_label:
|
||||
plt.legend(loc="upper center", fontsize=16)
|
||||
plt.axis(axes)
|
||||
|
||||
plt.figure(figsize=(11,11))
|
||||
|
||||
plt.subplot(321)
|
||||
plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h_1(x_1)$", style="g-", data_label="Training set")
|
||||
plt.ylabel("$y$", fontsize=16, rotation=0)
|
||||
plt.title("Residuals and tree predictions", fontsize=16)
|
||||
|
||||
plt.subplot(322)
|
||||
plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1)$", data_label="Training set")
|
||||
plt.ylabel("$y$", fontsize=16, rotation=0)
|
||||
plt.title("Ensemble predictions", fontsize=16)
|
||||
|
||||
plt.subplot(323)
|
||||
plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_2(x_1)$", style="g-", data_style="k+", data_label="Residuals")
|
||||
plt.ylabel("$y - h_1(x_1)$", fontsize=16)
|
||||
|
||||
plt.subplot(324)
|
||||
plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1)$")
|
||||
plt.ylabel("$y$", fontsize=16, rotation=0)
|
||||
|
||||
plt.subplot(325)
|
||||
plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_3(x_1)$", style="g-", data_style="k+")
|
||||
plt.ylabel("$y - h_1(x_1) - h_2(x_1)$", fontsize=16)
|
||||
plt.xlabel("$x_1$", fontsize=16)
|
||||
|
||||
plt.subplot(326)
|
||||
plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$")
|
||||
plt.xlabel("$x_1$", fontsize=16)
|
||||
plt.ylabel("$y$", fontsize=16, rotation=0)
|
||||
|
||||
save_fig("gradient_boosting_plot")
|
||||
plt.show()
|
||||
|
||||
from sklearn.ensemble import GradientBoostingRegressor
|
||||
|
||||
gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0, random_state=42)
|
||||
gbrt.fit(X, y)
|
||||
|
||||
gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)
|
||||
gbrt_slow.fit(X, y)
|
||||
|
||||
plt.figure(figsize=(11,4))
|
||||
|
||||
plt.subplot(121)
|
||||
plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="Ensemble predictions")
|
||||
plt.title("learning_rate={}, n_estimators={}".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)
|
||||
|
||||
plt.subplot(122)
|
||||
plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
|
||||
plt.title("learning_rate={}, n_estimators={}".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)
|
||||
|
||||
save_fig("gbrt_learning_rate_plot")
|
||||
plt.show()
|
||||
|
||||
!ec
|
||||
|
||||
Reference in New Issue
Block a user