+ + + + +
+ + +
from sklearn.model_selection import train_test_split
+from sklearn.metrics import mean_squared_error
+
+X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=49)
+
+gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=120, random_state=42)
+gbrt.fit(X_train, y_train)
+
+errors = [mean_squared_error(y_val, y_pred)
+ for y_pred in gbrt.staged_predict(X_val)]
+bst_n_estimators = np.argmin(errors) + 1
+
+gbrt_best = GradientBoostingRegressor(max_depth=2,n_estimators=bst_n_estimators, random_state=42)
+gbrt_best.fit(X_train, y_train)
+
+min_error = np.min(errors)
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.plot(errors, "b.-")
+plt.plot([bst_n_estimators, bst_n_estimators], [0, min_error], "k--")
+plt.plot([0, 120], [min_error, min_error], "k--")
+plt.plot(bst_n_estimators, min_error, "ko")
+plt.text(bst_n_estimators, min_error*1.2, "Minimum", ha="center", fontsize=14)
+plt.axis([0, 120, 0, 0.01])
+plt.xlabel("Number of trees")
+plt.title("Validation error", fontsize=14)
+
+plt.subplot(122)
+plot_predictions([gbrt_best], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
+plt.title("Best model (%d trees)" % bst_n_estimators, fontsize=14)
+
+save_fig("early_stopping_gbrt_plot")
+plt.show()
+
+
+gbrt = GradientBoostingRegressor(max_depth=2, warm_start=True, random_state=42)
+
+min_val_error = float("inf")
+error_going_up = 0
+for n_estimators in range(1, 120):
+ gbrt.n_estimators = n_estimators
+ gbrt.fit(X_train, y_train)
+ y_pred = gbrt.predict(X_val)
+ val_error = mean_squared_error(y_val, y_pred)
+ if val_error < min_val_error:
+ min_val_error = val_error
+ error_going_up = 0
+ else:
+ error_going_up += 1
+ if error_going_up == 5:
+ break # early stopping
+
+
+print(gbrt.n_estimators)
+print("Minimum validation MSE:", min_val_error)
++
+ +
+ + +
+ + + + +
+XGBoost or Extreme Gradient +Boosting, is an optimized distributed gradient boosting library +designed to be highly efficient, flexible and portable. It implements +machine learning algorithms under the Gradient Boosting +framework. XGBoost provides a parallel tree boosting that solve many +data science problems in a fast and accurate way. See the article by Chen and Guestrin. + +
+The authors design and build a highly scalable end-to-end tree +boosting system. It has a theoretically justified weighted quantile +sketch for efficient proposal calculation. It introduces a novel sparsity-aware algorithm for parallel tree learning and an effective cache-aware block structure for out-of-core tree learning. + +
+It is now the algorithm which wins essentially all ML competitions!!! + +
+
+ +
+ + +
+ + + + +
+ + +
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+import xgboost as xgb
+from sklearn.preprocessing import StandardScaler
+import scikitplot as skplt
+from sklearn.metrics import mean_squared_error
+
+n = 100
+maxdegree = 6
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = xgb.XGBRegressor(objective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1,
+ max_depth = degree, alpha = 10, n_estimators = 10)
+ model.fit(X_train_scaled,y_train)
+ y_pred = model.predict(X_test_scaled)
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
+ variance[degree] = np.mean( np.var(y_pred) )
+ print('Max depth:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.xlim(1,maxdegree-1)
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
++
+ +
+ + +
+ + + + +
+ + +
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+import scikitplot as skplt
+import xgboost as xgb
+# Load the data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+xg_clf = xgb.XGBClassifier()
+xg_clf.fit(X_train_scaled,y_train)
+xgb.plot_tree(xg_clf,num_trees=0)
+plt.rcParams['figure.figsize'] = [50, 10]
+plt.show()
+xgb.plot_importance(xg_clf)
+plt.rcParams['figure.figsize'] = [5, 5]
+plt.show()
++ +
+ +
+ + +