typos

doc/src/DecisionTrees/DecisionTrees.do.txt

@@ -1424,32 +1424,7 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
 !split
 ===== Feature Importance =====
 
-!bc pycod
-try:
-    from sklearn.datasets import fetch_openml
-    mnist = fetch_openml('mnist_784', version=1)
-    mnist.target = mnist.target.astype(np.int64)
-except ImportError:
-    from sklearn.datasets import fetch_mldata
-    mnist = fetch_mldata('MNIST original')
-
-rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
-rnd_clf.fit(mnist["data"], mnist["target"])
-
-def plot_digit(data):
-    image = data.reshape(28, 28)
-    plt.imshow(image, cmap = mpl.cm.hot,
-               interpolation="nearest")
-    plt.axis("off")
-
-plot_digit(rnd_clf.feature_importances_)
-
-cbar = plt.colorbar(ticks=[rnd_clf.feature_importances_.min(), rnd_clf.feature_importances_.max()])
-cbar.ax.set_yticklabels(['Not important', 'Very important'])
-
-#save_fig("mnist_feature_importance_plot")
-plt.show()
-!ec
+Example will be added here.
 
 !split
 ===== Boosting: AdaBoost =====
@@ -1579,3 +1554,66 @@ save_fig("gbrt_learning_rate_plot")
 plt.show()
 
 !ec
+
+
+!split
+===== Gradient Boots with Early Stopping =====
+!bc pycod
+
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import mean_squared_error
+
+X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=49)
+
+gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=120, random_state=42)
+gbrt.fit(X_train, y_train)
+
+errors = [mean_squared_error(y_val, y_pred)
+          for y_pred in gbrt.staged_predict(X_val)]
+bst_n_estimators = np.argmin(errors) + 1
+
+gbrt_best = GradientBoostingRegressor(max_depth=2,n_estimators=bst_n_estimators, random_state=42)
+gbrt_best.fit(X_train, y_train)
+
+min_error = np.min(errors)
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.plot(errors, "b.-")
+plt.plot([bst_n_estimators, bst_n_estimators], [0, min_error], "k--")
+plt.plot([0, 120], [min_error, min_error], "k--")
+plt.plot(bst_n_estimators, min_error, "ko")
+plt.text(bst_n_estimators, min_error*1.2, "Minimum", ha="center", fontsize=14)
+plt.axis([0, 120, 0, 0.01])
+plt.xlabel("Number of trees")
+plt.title("Validation error", fontsize=14)
+
+plt.subplot(122)
+plot_predictions([gbrt_best], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
+plt.title("Best model (%d trees)" % bst_n_estimators, fontsize=14)
+
+save_fig("early_stopping_gbrt_plot")
+plt.show()
+
+
+gbrt = GradientBoostingRegressor(max_depth=2, warm_start=True, random_state=42)
+
+min_val_error = float("inf")
+error_going_up = 0
+for n_estimators in range(1, 120):
+    gbrt.n_estimators = n_estimators
+    gbrt.fit(X_train, y_train)
+    y_pred = gbrt.predict(X_val)
+    val_error = mean_squared_error(y_val, y_pred)
+    if val_error < min_val_error:
+        min_val_error = val_error
+        error_going_up = 0
+    else:
+        error_going_up += 1
+        if error_going_up == 5:
+            break  # early stopping
+
+
+print(gbrt.n_estimators)
+print("Minimum validation MSE:", min_val_error)
+!ec