+See discussion during lecture November 8. +
@@ -286,6 +297,7 @@ function was the least squares function.
-
np.random.seed(42)
-X = np.random.rand(100, 1) - 0.5
-y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)
-
-from sklearn.tree import DecisionTreeRegressor
-
-tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg1.fit(X, y)
-
-y2 = y - tree_reg1.predict(X)
-tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg2.fit(X, y2)
-
-y3 = y2 - tree_reg2.predict(X)
-tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg3.fit(X, y3)
-
-X_new = np.array([[0.8]])
-y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))
-
-def plot_predictions(regressors, X, y, axes, label=None, style="r-", data_style="b.", data_label=None):
- x1 = np.linspace(axes[0], axes[1], 500)
- y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)
- plt.plot(X[:, 0], y, data_style, label=data_label)
- plt.plot(x1, y_pred, style, linewidth=2, label=label)
- if label or data_label:
- plt.legend(loc="upper center", fontsize=16)
- plt.axis(axes)
-
-plt.figure(figsize=(11,11))
-
-plt.subplot(321)
-plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h_1(x_1)$", style="g-", data_label="Training set")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-plt.title("Residuals and tree predictions", fontsize=16)
-
-plt.subplot(322)
-plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1)$", data_label="Training set")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-plt.title("Ensemble predictions", fontsize=16)
-
-plt.subplot(323)
-plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_2(x_1)$", style="g-", data_style="k+", data_label="Residuals")
-plt.ylabel("$y - h_1(x_1)$", fontsize=16)
-
-plt.subplot(324)
-plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1)$")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-
-plt.subplot(325)
-plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_3(x_1)$", style="g-", data_style="k+")
-plt.ylabel("$y - h_1(x_1) - h_2(x_1)$", fontsize=16)
-plt.xlabel("$x_1$", fontsize=16)
-
-plt.subplot(326)
-plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$")
-plt.xlabel("$x_1$", fontsize=16)
-plt.ylabel("$y$", fontsize=16, rotation=0)
-
-save_fig("gradient_boosting_plot")
-plt.show()
-
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingRegressor
+from sklearn.preprocessing import StandardScaler
+import scikitplot as skplt
+from sklearn.metrics import mean_squared_error
-gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0, random_state=42)
-gbrt.fit(X, y)
+n = 100
+maxdegree = 6
-gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)
-gbrt_slow.fit(X, y)
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
-plt.figure(figsize=(11,4))
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
-plt.subplot(121)
-plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="Ensemble predictions")
-plt.title("learning_rate={}, n_estimators={}".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)
+for degree in range(1,maxdegree):
+ model = GradientBoostingRegressor(max_depth=degree, n_estimators=100, learning_rate=1.0)
+ model.fit(X_train_scaled,y_train)
+ y_pred = model.predict(X_test_scaled)
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
+ variance[degree] = np.mean( np.var(y_pred) )
+ print('Max depth:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
-plt.subplot(122)
-plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
-plt.title("learning_rate={}, n_estimators={}".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)
-
-save_fig("gbrt_learning_rate_plot")
+plt.xlim(1,maxdegree-1)
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
plt.show()
@@ -359,6 +330,7 @@ plt.show()
-XGBoost or Extreme Gradient -Boosting, is an optimized distributed gradient boosting library -designed to be highly efficient, flexible and portable. It implements -machine learning algorithms under the Gradient Boosting -framework. XGBoost provides a parallel tree boosting that solve many -data science problems in a fast and accurate way. See the article by Chen and Guestrin. -
-The authors design and build a highly scalable end-to-end tree -boosting system. It has a theoretically justified weighted quantile -sketch for efficient proposal calculation. It introduces a novel sparsity-aware algorithm for parallel tree learning and an effective cache-aware block structure for out-of-core tree learning. + +
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+import scikitplot as skplt
+from sklearn.ensemble import GradientBoostingClassifier
+from sklearn.model_selection import cross_validate
-
-It is now the algorithm which wins essentially all ML competitions!!!
+# Load the data
+cancer = load_breast_cancer()
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+gd_clf = GradientBoostingClassifier(max_depth=3, n_estimators=100, learning_rate=1.0)
+gd_clf.fit(X_train_scaled, y_train)
+#Cross validation
+accuracy = cross_validate(gd_clf,X_test_scaled,y_test,cv=10)['test_score']
+print(accuracy)
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(gd_clf.score(X_test_scaled,y_test)))
+
+import scikitplot as skplt
+y_pred = gd_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = gd_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
+
@@ -289,6 +321,7 @@ It is now the algorithm which wins essentially all ML competitions!!!
$$
-\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(\boldsymbol{X}_{i*})}{\sum_{i=0}^{n-1}w_i^m},
+\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i)}{\sum_{i=0}^{n-1}w_i^m},
$$
@@ -2207,7 +2207,7 @@ The basic idea is to set up weights which will be used to scale the correctly cl
$$
-\mathrm{err}=\frac{\sum_{i=0}^{n-1}w_iI(y_i\ne G(x_i})}{\sum_{i=0}^{n-1}w_i},
+\mathrm{\overline{err}}_m=\frac{\sum_{i=0}^{n-1}w_i^m I(y_i\ne G(x_i))}{\sum_{i=0}^{n-1}w_i},
$$
@@ -2218,7 +2218,7 @@ $$
+See discussion during lecture November 8. @@ -2303,7 +2306,7 @@ The way we proceed in an iterative fashion is to
-
np.random.seed(42)
-X = np.random.rand(100, 1) - 0.5
-y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)
-
-from sklearn.tree import DecisionTreeRegressor
-
-tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg1.fit(X, y)
-
-y2 = y - tree_reg1.predict(X)
-tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg2.fit(X, y2)
-
-y3 = y2 - tree_reg2.predict(X)
-tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg3.fit(X, y3)
-
-X_new = np.array([[0.8]])
-y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))
-
-def plot_predictions(regressors, X, y, axes, label=None, style="r-", data_style="b.", data_label=None):
- x1 = np.linspace(axes[0], axes[1], 500)
- y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)
- plt.plot(X[:, 0], y, data_style, label=data_label)
- plt.plot(x1, y_pred, style, linewidth=2, label=label)
- if label or data_label:
- plt.legend(loc="upper center", fontsize=16)
- plt.axis(axes)
-
-plt.figure(figsize=(11,11))
-
-plt.subplot(321)
-plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h_1(x_1)$", style="g-", data_label="Training set")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-plt.title("Residuals and tree predictions", fontsize=16)
-
-plt.subplot(322)
-plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1)$", data_label="Training set")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-plt.title("Ensemble predictions", fontsize=16)
-
-plt.subplot(323)
-plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_2(x_1)$", style="g-", data_style="k+", data_label="Residuals")
-plt.ylabel("$y - h_1(x_1)$", fontsize=16)
-
-plt.subplot(324)
-plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1)$")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-
-plt.subplot(325)
-plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_3(x_1)$", style="g-", data_style="k+")
-plt.ylabel("$y - h_1(x_1) - h_2(x_1)$", fontsize=16)
-plt.xlabel("$x_1$", fontsize=16)
-
-plt.subplot(326)
-plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$")
-plt.xlabel("$x_1$", fontsize=16)
-plt.ylabel("$y$", fontsize=16, rotation=0)
-
-save_fig("gradient_boosting_plot")
-plt.show()
-
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingRegressor
+from sklearn.preprocessing import StandardScaler
+import scikitplot as skplt
+from sklearn.metrics import mean_squared_error
-gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0, random_state=42)
-gbrt.fit(X, y)
+n = 100
+maxdegree = 6
-gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)
-gbrt_slow.fit(X, y)
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
-plt.figure(figsize=(11,4))
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
-plt.subplot(121)
-plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="Ensemble predictions")
-plt.title("learning_rate={}, n_estimators={}".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)
+for degree in range(1,maxdegree):
+ model = GradientBoostingRegressor(max_depth=degree, n_estimators=100, learning_rate=1.0)
+ model.fit(X_train_scaled,y_train)
+ y_pred = model.predict(X_test_scaled)
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
+ variance[degree] = np.mean( np.var(y_pred) )
+ print('Max depth:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
-plt.subplot(122)
-plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
-plt.title("learning_rate={}, n_estimators={}".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)
-
-save_fig("gbrt_learning_rate_plot")
+plt.xlim(1,maxdegree-1)
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
plt.show()
-XGBoost: Extreme Gradient Boosting
+Gradient Boosting, Examples of Classification
+
+
+
+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+import scikitplot as skplt
+from sklearn.ensemble import GradientBoostingClassifier
+from sklearn.model_selection import cross_validate
+
+# Load the data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+gd_clf = GradientBoostingClassifier(max_depth=3, n_estimators=100, learning_rate=1.0)
+gd_clf.fit(X_train_scaled, y_train)
+#Cross validation
+accuracy = cross_validate(gd_clf,X_test_scaled,y_test,cv=10)['test_score']
+print(accuracy)
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(gd_clf.score(X_test_scaled,y_test)))
+
+import scikitplot as skplt
+y_pred = gd_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = gd_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
+
+
+
+
+
+XGBoost: Extreme Gradient Boosting
XGBoost or Extreme Gradient
@@ -2425,7 +2437,7 @@ It is now the algorithm which wins essentially all ML competitions!!!
-Regression Case
+Regression Case
@@ -2456,8 +2468,8 @@ X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)
for degree in range(maxdegree):
- model = xgb.XGBRegressor(objective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1,
- max_depth = degree, alpha = 10, n_estimators = 10)
+ model = xgb.XGBRegressor(objective ='reg:squarederror', colsaobjective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1,max_depth = degree, alpha = 10, n_estimators = 200)
+
model.fit(X_train_scaled,y_train)
y_pred = model.predict(X_test_scaled)
polydegree[degree] = degree
@@ -2481,7 +2493,7 @@ plt.show()
-Xgboost on the Cancer Data
+Xgboost on the Cancer Data
@@ -2508,9 +2520,26 @@ X_test_scaled = scaler.transform(X_test)
xg_clf = xgb.XGBClassifier()
xg_clf.fit(X_train_scaled,y_train)
+
+y_test = xg_clf.predict(X_test_scaled)
+
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(xg_clf.score(X_test_scaled,y_test)))
+
+import scikitplot as skplt
+y_pred = xg_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = xg_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
+
+
xgb.plot_tree(xg_clf,num_trees=0)
plt.rcParams['figure.figsize'] = [50, 10]
plt.show()
+
xgb.plot_importance(xg_clf)
plt.rcParams['figure.figsize'] = [5, 5]
plt.show()
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html b/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html
index 8c73017c1..70eb4082b 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html
@@ -158,10 +158,17 @@ div { text-align: justify; text-justify: inter-word; }
('AdaBoost Examples', 2, None, '___sec52'),
('Gradient boosting: Basics', 2, None, '___sec53'),
('Gradient Boosting, algorithm', 2, None, '___sec54'),
- ('Gradient Boosting, Examples', 2, None, '___sec55'),
- ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec56'),
- ('Regression Case', 2, None, '___sec57'),
- ('Xgboost on the Cancer Data', 2, None, '___sec58')]}
+ ('Gradient Boosting, Examples of Regression',
+ 2,
+ None,
+ '___sec55'),
+ ('Gradient Boosting, Examples of Classification',
+ 2,
+ None,
+ '___sec56'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec57'),
+ ('Regression Case', 2, None, '___sec58'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec59')]}
end of tocinfo -->
@@ -2120,7 +2127,7 @@ $$
where we have redefined the error as
$$
-\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(\boldsymbol{X}_{i*})}{\sum_{i=0}^{n-1}w_i^m},
+\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i)}{\sum_{i=0}^{n-1}w_i^m},
$$
which leads to an update of
@@ -2169,7 +2176,7 @@ The basic idea is to set up weights which will be used to scale the correctly cl
$$
-\mathrm{err}=\frac{\sum_{i=0}^{n-1}w_iI(y_i\ne G(x_i})}{\sum_{i=0}^{n-1}w_i},
+\mathrm{\overline{err}}_m=\frac{\sum_{i=0}^{n-1}w_i^m I(y_i\ne G(x_i))}{\sum_{i=0}^{n-1}w_i},
$$
@@ -2179,7 +2186,7 @@ $$
- Fit then a given classifier to the training using the weights \( w_i \).
- Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.
- - Define a quantity \( \alpha_{m} = \log{(1-\mathrm{err})/\mathrm{err}} \)
+ - Define a quantity \( \alpha_{m} = \log{(1-\mathrm{\overline{err}}_m)/\mathrm{\overline{err}}_m} \)
- Set the new weights to \( w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(x_i)} \).
@@ -2242,6 +2249,9 @@ In order to understand the method, let us illustrate its basics by
bringing back the essential steps in linear regression, where our cost
function was the least squares function.
+
+See discussion during lecture November 8.
+
@@ -2261,7 +2271,7 @@ The way we proceed in an iterative fashion is to
For \( m=1:M \), we
- compute the negative gradient vector \( \boldsymbol{u}_m = -\partial C(\boldsymbol{y},\boldsymbol{f})/\partial \boldsymbol{f}(x) \) at $f(x) = f_{m-1}(x);
+ compute the negative gradient vector \( \boldsymbol{u}_m = -\partial C(\boldsymbol{y},\boldsymbol{f})/\partial \boldsymbol{f}(x) \) at \( f(x) = f_{m-1}(x) \);
fit the so-called base-learner to the negative gradient \( h_m(u_m,x) \);
update the estimate \( f_m(x) = f_{m-1}(x)+\nu h_m(u_m,x) \);
@@ -2271,97 +2281,105 @@ The way we proceed in an iterative fashion is to
-Gradient Boosting, Examples
+Gradient Boosting, Examples of Regression
-
np.random.seed(42)
-X = np.random.rand(100, 1) - 0.5
-y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)
-
-from sklearn.tree import DecisionTreeRegressor
-
-tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg1.fit(X, y)
-
-y2 = y - tree_reg1.predict(X)
-tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg2.fit(X, y2)
-
-y3 = y2 - tree_reg2.predict(X)
-tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg3.fit(X, y3)
-
-X_new = np.array([[0.8]])
-y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))
-
-def plot_predictions(regressors, X, y, axes, label=None, style="r-", data_style="b.", data_label=None):
- x1 = np.linspace(axes[0], axes[1], 500)
- y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)
- plt.plot(X[:, 0], y, data_style, label=data_label)
- plt.plot(x1, y_pred, style, linewidth=2, label=label)
- if label or data_label:
- plt.legend(loc="upper center", fontsize=16)
- plt.axis(axes)
-
-plt.figure(figsize=(11,11))
-
-plt.subplot(321)
-plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h_1(x_1)$", style="g-", data_label="Training set")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-plt.title("Residuals and tree predictions", fontsize=16)
-
-plt.subplot(322)
-plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1)$", data_label="Training set")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-plt.title("Ensemble predictions", fontsize=16)
-
-plt.subplot(323)
-plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_2(x_1)$", style="g-", data_style="k+", data_label="Residuals")
-plt.ylabel("$y - h_1(x_1)$", fontsize=16)
-
-plt.subplot(324)
-plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1)$")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-
-plt.subplot(325)
-plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_3(x_1)$", style="g-", data_style="k+")
-plt.ylabel("$y - h_1(x_1) - h_2(x_1)$", fontsize=16)
-plt.xlabel("$x_1$", fontsize=16)
-
-plt.subplot(326)
-plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$")
-plt.xlabel("$x_1$", fontsize=16)
-plt.ylabel("$y$", fontsize=16, rotation=0)
-
-save_fig("gradient_boosting_plot")
-plt.show()
-
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingRegressor
+from sklearn.preprocessing import StandardScaler
+import scikitplot as skplt
+from sklearn.metrics import mean_squared_error
-gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0, random_state=42)
-gbrt.fit(X, y)
+n = 100
+maxdegree = 6
-gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)
-gbrt_slow.fit(X, y)
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
-plt.figure(figsize=(11,4))
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
-plt.subplot(121)
-plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="Ensemble predictions")
-plt.title("learning_rate={}, n_estimators={}".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)
+for degree in range(1,maxdegree):
+ model = GradientBoostingRegressor(max_depth=degree, n_estimators=100, learning_rate=1.0)
+ model.fit(X_train_scaled,y_train)
+ y_pred = model.predict(X_test_scaled)
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
+ variance[degree] = np.mean( np.var(y_pred) )
+ print('Max depth:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
-plt.subplot(122)
-plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
-plt.title("learning_rate={}, n_estimators={}".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)
-
-save_fig("gbrt_learning_rate_plot")
+plt.xlim(1,maxdegree-1)
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
plt.show()
-
XGBoost: Extreme Gradient Boosting
+Gradient Boosting, Examples of Classification
+
+
+
+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+import scikitplot as skplt
+from sklearn.ensemble import GradientBoostingClassifier
+from sklearn.model_selection import cross_validate
+
+# Load the data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+gd_clf = GradientBoostingClassifier(max_depth=3, n_estimators=100, learning_rate=1.0)
+gd_clf.fit(X_train_scaled, y_train)
+#Cross validation
+accuracy = cross_validate(gd_clf,X_test_scaled,y_test,cv=10)['test_score']
+print(accuracy)
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(gd_clf.score(X_test_scaled,y_test)))
+
+import scikitplot as skplt
+y_pred = gd_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = gd_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
+
+
+
+
+
XGBoost: Extreme Gradient Boosting
XGBoost or Extreme Gradient
@@ -2382,7 +2400,7 @@ It is now the algorithm which wins essentially all ML competitions!!!
-
Regression Case
+Regression Case
@@ -2413,8 +2431,8 @@ X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)
for degree in range(maxdegree):
- model = xgb.XGBRegressor(objective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1,
- max_depth = degree, alpha = 10, n_estimators = 10)
+ model = xgb.XGBRegressor(objective ='reg:squarederror', colsaobjective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1,max_depth = degree, alpha = 10, n_estimators = 200)
+
model.fit(X_train_scaled,y_train)
y_pred = model.predict(X_test_scaled)
polydegree[degree] = degree
@@ -2437,7 +2455,7 @@ plt.show()
-
Xgboost on the Cancer Data
+Xgboost on the Cancer Data
@@ -2464,9 +2482,26 @@ X_test_scaled = scaler.transform(X_test)
xg_clf = xgb.XGBClassifier()
xg_clf.fit(X_train_scaled,y_train)
+
+y_test = xg_clf.predict(X_test_scaled)
+
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(xg_clf.score(X_test_scaled,y_test)))
+
+import scikitplot as skplt
+y_pred = xg_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = xg_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
+
+
xgb.plot_tree(xg_clf,num_trees=0)
plt.rcParams['figure.figsize'] = [50, 10]
plt.show()
+
xgb.plot_importance(xg_clf)
plt.rcParams['figure.figsize'] = [5, 5]
plt.show()
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees.html b/doc/pub/DecisionTrees/html/DecisionTrees.html
index a396507d9..a0192166f 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees.html
@@ -163,10 +163,17 @@ div { text-align: justify; text-justify: inter-word; }
('AdaBoost Examples', 2, None, '___sec52'),
('Gradient boosting: Basics', 2, None, '___sec53'),
('Gradient Boosting, algorithm', 2, None, '___sec54'),
- ('Gradient Boosting, Examples', 2, None, '___sec55'),
- ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec56'),
- ('Regression Case', 2, None, '___sec57'),
- ('Xgboost on the Cancer Data', 2, None, '___sec58')]}
+ ('Gradient Boosting, Examples of Regression',
+ 2,
+ None,
+ '___sec55'),
+ ('Gradient Boosting, Examples of Classification',
+ 2,
+ None,
+ '___sec56'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec57'),
+ ('Regression Case', 2, None, '___sec58'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec59')]}
end of tocinfo -->
@@ -2125,7 +2132,7 @@ $$
where we have redefined the error as
$$
-\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(\boldsymbol{X}_{i*})}{\sum_{i=0}^{n-1}w_i^m},
+\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i)}{\sum_{i=0}^{n-1}w_i^m},
$$
which leads to an update of
@@ -2174,7 +2181,7 @@ The basic idea is to set up weights which will be used to scale the correctly cl
$$
-\mathrm{err}=\frac{\sum_{i=0}^{n-1}w_iI(y_i\ne G(x_i})}{\sum_{i=0}^{n-1}w_i},
+\mathrm{\overline{err}}_m=\frac{\sum_{i=0}^{n-1}w_i^m I(y_i\ne G(x_i))}{\sum_{i=0}^{n-1}w_i},
$$
@@ -2184,7 +2191,7 @@ $$
- Fit then a given classifier to the training using the weights \( w_i \).
- Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.
- - Define a quantity \( \alpha_{m} = \log{(1-\mathrm{err})/\mathrm{err}} \)
+ - Define a quantity \( \alpha_{m} = \log{(1-\mathrm{\overline{err}}_m)/\mathrm{\overline{err}}_m} \)
- Set the new weights to \( w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(x_i)} \).
@@ -2247,6 +2254,9 @@ In order to understand the method, let us illustrate its basics by
bringing back the essential steps in linear regression, where our cost
function was the least squares function.
+
+See discussion during lecture November 8.
+
@@ -2266,7 +2276,7 @@ The way we proceed in an iterative fashion is to
For \( m=1:M \), we
- compute the negative gradient vector \( \boldsymbol{u}_m = -\partial C(\boldsymbol{y},\boldsymbol{f})/\partial \boldsymbol{f}(x) \) at $f(x) = f_{m-1}(x);
+ compute the negative gradient vector \( \boldsymbol{u}_m = -\partial C(\boldsymbol{y},\boldsymbol{f})/\partial \boldsymbol{f}(x) \) at \( f(x) = f_{m-1}(x) \);
fit the so-called base-learner to the negative gradient \( h_m(u_m,x) \);
update the estimate \( f_m(x) = f_{m-1}(x)+\nu h_m(u_m,x) \);
@@ -2276,97 +2286,105 @@ The way we proceed in an iterative fashion is to
-Gradient Boosting, Examples
+Gradient Boosting, Examples of Regression
-
np.random.seed(42)
-X = np.random.rand(100, 1) - 0.5
-y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)
-
-from sklearn.tree import DecisionTreeRegressor
-
-tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg1.fit(X, y)
-
-y2 = y - tree_reg1.predict(X)
-tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg2.fit(X, y2)
-
-y3 = y2 - tree_reg2.predict(X)
-tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg3.fit(X, y3)
-
-X_new = np.array([[0.8]])
-y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))
-
-def plot_predictions(regressors, X, y, axes, label=None, style="r-", data_style="b.", data_label=None):
- x1 = np.linspace(axes[0], axes[1], 500)
- y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)
- plt.plot(X[:, 0], y, data_style, label=data_label)
- plt.plot(x1, y_pred, style, linewidth=2, label=label)
- if label or data_label:
- plt.legend(loc="upper center", fontsize=16)
- plt.axis(axes)
-
-plt.figure(figsize=(11,11))
-
-plt.subplot(321)
-plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h_1(x_1)$", style="g-", data_label="Training set")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-plt.title("Residuals and tree predictions", fontsize=16)
-
-plt.subplot(322)
-plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1)$", data_label="Training set")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-plt.title("Ensemble predictions", fontsize=16)
-
-plt.subplot(323)
-plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_2(x_1)$", style="g-", data_style="k+", data_label="Residuals")
-plt.ylabel("$y - h_1(x_1)$", fontsize=16)
-
-plt.subplot(324)
-plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1)$")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-
-plt.subplot(325)
-plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_3(x_1)$", style="g-", data_style="k+")
-plt.ylabel("$y - h_1(x_1) - h_2(x_1)$", fontsize=16)
-plt.xlabel("$x_1$", fontsize=16)
-
-plt.subplot(326)
-plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$")
-plt.xlabel("$x_1$", fontsize=16)
-plt.ylabel("$y$", fontsize=16, rotation=0)
-
-save_fig("gradient_boosting_plot")
-plt.show()
-
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingRegressor
+from sklearn.preprocessing import StandardScaler
+import scikitplot as skplt
+from sklearn.metrics import mean_squared_error
-gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0, random_state=42)
-gbrt.fit(X, y)
+n = 100
+maxdegree = 6
-gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)
-gbrt_slow.fit(X, y)
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
-plt.figure(figsize=(11,4))
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
-plt.subplot(121)
-plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="Ensemble predictions")
-plt.title("learning_rate={}, n_estimators={}".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)
+for degree in range(1,maxdegree):
+ model = GradientBoostingRegressor(max_depth=degree, n_estimators=100, learning_rate=1.0)
+ model.fit(X_train_scaled,y_train)
+ y_pred = model.predict(X_test_scaled)
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
+ variance[degree] = np.mean( np.var(y_pred) )
+ print('Max depth:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
-plt.subplot(122)
-plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
-plt.title("learning_rate={}, n_estimators={}".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)
-
-save_fig("gbrt_learning_rate_plot")
+plt.xlim(1,maxdegree-1)
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
plt.show()
-
XGBoost: Extreme Gradient Boosting
+Gradient Boosting, Examples of Classification
+
+
+
+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+import scikitplot as skplt
+from sklearn.ensemble import GradientBoostingClassifier
+from sklearn.model_selection import cross_validate
+
+# Load the data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+gd_clf = GradientBoostingClassifier(max_depth=3, n_estimators=100, learning_rate=1.0)
+gd_clf.fit(X_train_scaled, y_train)
+#Cross validation
+accuracy = cross_validate(gd_clf,X_test_scaled,y_test,cv=10)['test_score']
+print(accuracy)
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(gd_clf.score(X_test_scaled,y_test)))
+
+import scikitplot as skplt
+y_pred = gd_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = gd_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
+
+
+
+
+
XGBoost: Extreme Gradient Boosting
XGBoost or Extreme Gradient
@@ -2387,7 +2405,7 @@ It is now the algorithm which wins essentially all ML competitions!!!
-
Regression Case
+Regression Case
@@ -2418,8 +2436,8 @@ X_train_scaled = scaler= scaler.transform(X_test)
for degree in range(maxdegree):
- model = xgb.XGBRegressor(objective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1,
- max_depth = degree, alpha = 10, n_estimators = 10)
+ model = xgb.XGBRegressor(objective ='reg:squarederror', colsaobjective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1,max_depth = degree, alpha = 10, n_estimators = 200)
+
model.fit(X_train_scaled,y_train)
y_pred = model.predict(X_test_scaled)
polydegree[degree] = degree
@@ -2442,7 +2460,7 @@ plt.show()
-
Xgboost on the Cancer Data
+Xgboost on the Cancer Data
@@ -2469,9 +2487,26 @@ X_test_scaled = scaler= xgb.XGBClassifier()
xg_clf.fit(X_train_scaled,y_train)
+
+y_test = xg_clf.predict(X_test_scaled)
+
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(xg_clf.score(X_test_scaled,y_test)))
+
+import scikitplot as skplt
+y_pred = xg_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = xg_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
+
+
xgb.plot_tree(xg_clf,num_trees=0)
plt.rcParams['figure.figsize'] = [50, 10]
plt.show()
+
xgb.plot_importance(xg_clf)
plt.rcParams['figure.figsize'] = [5, 5]
plt.show()
diff --git a/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb b/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
index 2beee1adf..bb4157450 100644
--- a/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
+++ b/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
@@ -2283,7 +2283,7 @@
"metadata": {},
"source": [
"$$\n",
- "\\mathrm{\\overline{err}}_m=\\frac{1}{n}\\frac{\\sum_{i=0}^{n-1}w_i^mI(y_i\\ne G(\\boldsymbol{X}_{i*})}{\\sum_{i=0}^{n-1}w_i^m},\n",
+ "\\mathrm{\\overline{err}}_m=\\frac{1}{n}\\frac{\\sum_{i=0}^{n-1}w_i^mI(y_i\\ne G(x_i)}{\\sum_{i=0}^{n-1}w_i^m},\n",
"$$"
]
},
@@ -2365,7 +2365,7 @@
"metadata": {},
"source": [
"$$\n",
- "\\mathrm{err}=\\frac{\\sum_{i=0}^{n-1}w_iI(y_i\\ne G(x_i})}{\\sum_{i=0}^{n-1}w_i},\n",
+ "\\mathrm{\\overline{err}}_m=\\frac{\\sum_{i=0}^{n-1}w_i^m I(y_i\\ne G(x_i))}{\\sum_{i=0}^{n-1}w_i},\n",
"$$"
]
},
@@ -2379,7 +2379,7 @@
"\n",
"b. Compute then $\\mathrm{err}$ and figure out which events are classified properly and which are classified wrongly.\n",
"\n",
- "c. Define a quantity $\\alpha_{m} = \\log{(1-\\mathrm{err})/\\mathrm{err}}$\n",
+ "c. Define a quantity $\\alpha_{m} = \\log{(1-\\mathrm{\\overline{err}}_m)/\\mathrm{\\overline{err}}_m}$\n",
"\n",
"d. Set the new weights to $w_i = w_i\\times \\exp{(\\alpha_m I(y_i\\ne G(x_i)}$.\n",
"\n",
@@ -2446,6 +2446,8 @@
"bringing back the essential steps in linear regression, where our cost\n",
"function was the least squares function.\n",
"\n",
+ "See discussion during lecture November 8.\n",
+ "\n",
"## Gradient Boosting, algorithm\n",
"\n",
"Suppose we have a cost function $C(f)=\\sum_{i=0}^{n-1}L(y_i, f(x_i))$ where $y_i$ is our target and $f(x_i)$ the function which is meant to model $y_i$. The above cost function could be our standard square-error function"
@@ -2469,7 +2471,7 @@
"\n",
"2. For $m=1:M$, we\n",
"\n",
- "a. compute the negative gradient vector $\\boldsymbol{u}_m = -\\partial C(\\boldsymbol{y},\\boldsymbol{f})/\\partial \\boldsymbol{f}(x)$ at $f(x) = f_{m-1}(x);\n",
+ "a. compute the negative gradient vector $\\boldsymbol{u}_m = -\\partial C(\\boldsymbol{y},\\boldsymbol{f})/\\partial \\boldsymbol{f}(x)$ at $f(x) = f_{m-1}(x)$;\n",
"\n",
"b. fit the so-called base-learner to the negative gradient $h_m(u_m,x)$;\n",
"\n",
@@ -2478,7 +2480,7 @@
"\n",
"4. The final estimate is then $f_M(x) = \\sum_{m=1}^M\\nu h_m(u_m,x)$.\n",
"\n",
- "## Gradient Boosting, Examples"
+ "## Gradient Boosting, Examples of Regression"
]
},
{
@@ -2489,87 +2491,104 @@
},
"outputs": [],
"source": [
- "np.random.seed(42)\n",
- "X = np.random.rand(100, 1) - 0.5\n",
- "y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)\n",
- "\n",
- "from sklearn.tree import DecisionTreeRegressor\n",
- "\n",
- "tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)\n",
- "tree_reg1.fit(X, y)\n",
- "\n",
- "y2 = y - tree_reg1.predict(X)\n",
- "tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)\n",
- "tree_reg2.fit(X, y2)\n",
- "\n",
- "y3 = y2 - tree_reg2.predict(X)\n",
- "tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)\n",
- "tree_reg3.fit(X, y3)\n",
- "\n",
- "X_new = np.array([[0.8]])\n",
- "y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))\n",
- "\n",
- "def plot_predictions(regressors, X, y, axes, label=None, style=\"r-\", data_style=\"b.\", data_label=None):\n",
- " x1 = np.linspace(axes[0], axes[1], 500)\n",
- " y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)\n",
- " plt.plot(X[:, 0], y, data_style, label=data_label)\n",
- " plt.plot(x1, y_pred, style, linewidth=2, label=label)\n",
- " if label or data_label:\n",
- " plt.legend(loc=\"upper center\", fontsize=16)\n",
- " plt.axis(axes)\n",
- "\n",
- "plt.figure(figsize=(11,11))\n",
- "\n",
- "plt.subplot(321)\n",
- "plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h_1(x_1)$\", style=\"g-\", data_label=\"Training set\")\n",
- "plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n",
- "plt.title(\"Residuals and tree predictions\", fontsize=16)\n",
- "\n",
- "plt.subplot(322)\n",
- "plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h(x_1) = h_1(x_1)$\", data_label=\"Training set\")\n",
- "plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n",
- "plt.title(\"Ensemble predictions\", fontsize=16)\n",
- "\n",
- "plt.subplot(323)\n",
- "plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label=\"$h_2(x_1)$\", style=\"g-\", data_style=\"k+\", data_label=\"Residuals\")\n",
- "plt.ylabel(\"$y - h_1(x_1)$\", fontsize=16)\n",
- "\n",
- "plt.subplot(324)\n",
- "plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h(x_1) = h_1(x_1) + h_2(x_1)$\")\n",
- "plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n",
- "\n",
- "plt.subplot(325)\n",
- "plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label=\"$h_3(x_1)$\", style=\"g-\", data_style=\"k+\")\n",
- "plt.ylabel(\"$y - h_1(x_1) - h_2(x_1)$\", fontsize=16)\n",
- "plt.xlabel(\"$x_1$\", fontsize=16)\n",
- "\n",
- "plt.subplot(326)\n",
- "plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$\")\n",
- "plt.xlabel(\"$x_1$\", fontsize=16)\n",
- "plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n",
- "\n",
- "save_fig(\"gradient_boosting_plot\")\n",
- "plt.show()\n",
- "\n",
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "from sklearn.model_selection import train_test_split\n",
"from sklearn.ensemble import GradientBoostingRegressor\n",
+ "from sklearn.preprocessing import StandardScaler\n",
+ "import scikitplot as skplt\n",
+ "from sklearn.metrics import mean_squared_error\n",
"\n",
- "gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0, random_state=42)\n",
- "gbrt.fit(X, y)\n",
+ "n = 100\n",
+ "maxdegree = 6\n",
"\n",
- "gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)\n",
- "gbrt_slow.fit(X, y)\n",
+ "# Make data set.\n",
+ "x = np.linspace(-3, 3, n).reshape(-1, 1)\n",
+ "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n",
"\n",
- "plt.figure(figsize=(11,4))\n",
+ "error = np.zeros(maxdegree)\n",
+ "bias = np.zeros(maxdegree)\n",
+ "variance = np.zeros(maxdegree)\n",
+ "polydegree = np.zeros(maxdegree)\n",
+ "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
+ "scaler = StandardScaler()\n",
+ "scaler.fit(X_train)\n",
+ "X_train_scaled = scaler.transform(X_train)\n",
+ "X_test_scaled = scaler.transform(X_test)\n",
"\n",
- "plt.subplot(121)\n",
- "plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"Ensemble predictions\")\n",
- "plt.title(\"learning_rate={}, n_estimators={}\".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)\n",
+ "for degree in range(1,maxdegree):\n",
+ " model = GradientBoostingRegressor(max_depth=degree, n_estimators=100, learning_rate=1.0) \n",
+ " model.fit(X_train_scaled,y_train)\n",
+ " y_pred = model.predict(X_test_scaled)\n",
+ " polydegree[degree] = degree\n",
+ " error[degree] = np.mean( np.mean((y_test - y_pred)**2) )\n",
+ " bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )\n",
+ " variance[degree] = np.mean( np.var(y_pred) )\n",
+ " print('Max depth:', degree)\n",
+ " print('Error:', error[degree])\n",
+ " print('Bias^2:', bias[degree])\n",
+ " print('Var:', variance[degree])\n",
+ " print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))\n",
"\n",
- "plt.subplot(122)\n",
- "plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])\n",
- "plt.title(\"learning_rate={}, n_estimators={}\".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)\n",
+ "plt.xlim(1,maxdegree-1)\n",
+ "plt.plot(polydegree, error, label='Error')\n",
+ "plt.plot(polydegree, bias, label='bias')\n",
+ "plt.plot(polydegree, variance, label='Variance')\n",
+ "plt.legend()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Gradient Boosting, Examples of Classification"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "from sklearn.model_selection import train_test_split \n",
+ "from sklearn.datasets import load_breast_cancer\n",
+ "import scikitplot as skplt\n",
+ "from sklearn.ensemble import GradientBoostingClassifier\n",
+ "from sklearn.model_selection import cross_validate\n",
"\n",
- "save_fig(\"gbrt_learning_rate_plot\")\n",
+ "# Load the data\n",
+ "cancer = load_breast_cancer()\n",
+ "\n",
+ "X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)\n",
+ "print(X_train.shape)\n",
+ "print(X_test.shape)\n",
+ "#now scale the data\n",
+ "from sklearn.preprocessing import StandardScaler\n",
+ "scaler = StandardScaler()\n",
+ "scaler.fit(X_train)\n",
+ "X_train_scaled = scaler.transform(X_train)\n",
+ "X_test_scaled = scaler.transform(X_test)\n",
+ "\n",
+ "gd_clf = GradientBoostingClassifier(max_depth=3, n_estimators=100, learning_rate=1.0) \n",
+ "gd_clf.fit(X_train_scaled, y_train)\n",
+ "#Cross validation\n",
+ "accuracy = cross_validate(gd_clf,X_test_scaled,y_test,cv=10)['test_score']\n",
+ "print(accuracy)\n",
+ "print(\"Test set accuracy with Random Forests and scaled data: {:.2f}\".format(gd_clf.score(X_test_scaled,y_test)))\n",
+ "\n",
+ "import scikitplot as skplt\n",
+ "y_pred = gd_clf.predict(X_test_scaled)\n",
+ "skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)\n",
+ "plt.show()\n",
+ "y_probas = gd_clf.predict_proba(X_test_scaled)\n",
+ "skplt.metrics.plot_roc(y_test, y_probas)\n",
+ "plt.show()\n",
+ "skplt.metrics.plot_cumulative_gain(y_test, y_probas)\n",
"plt.show()"
]
},
@@ -2598,7 +2617,7 @@
},
{
"cell_type": "code",
- "execution_count": 30,
+ "execution_count": 31,
"metadata": {
"collapsed": false
},
@@ -2630,8 +2649,8 @@
"X_test_scaled = scaler.transform(X_test)\n",
"\n",
"for degree in range(maxdegree):\n",
- " model = xgb.XGBRegressor(objective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1,\n",
- " max_depth = degree, alpha = 10, n_estimators = 10)\n",
+ " model = xgb.XGBRegressor(objective ='reg:squarederror', colsaobjective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1,max_depth = degree, alpha = 10, n_estimators = 200)\n",
+ "\n",
" model.fit(X_train_scaled,y_train)\n",
" y_pred = model.predict(X_test_scaled)\n",
" polydegree[degree] = degree\n",
@@ -2661,12 +2680,13 @@
},
{
"cell_type": "code",
- "execution_count": 31,
+ "execution_count": 32,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
+ "\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"from sklearn.model_selection import train_test_split \n",
@@ -2690,9 +2710,26 @@
"\n",
"xg_clf = xgb.XGBClassifier()\n",
"xg_clf.fit(X_train_scaled,y_train)\n",
+ "\n",
+ "y_test = xg_clf.predict(X_test_scaled)\n",
+ "\n",
+ "print(\"Test set accuracy with Random Forests and scaled data: {:.2f}\".format(xg_clf.score(X_test_scaled,y_test)))\n",
+ "\n",
+ "import scikitplot as skplt\n",
+ "y_pred = xg_clf.predict(X_test_scaled)\n",
+ "skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)\n",
+ "plt.show()\n",
+ "y_probas = xg_clf.predict_proba(X_test_scaled)\n",
+ "skplt.metrics.plot_roc(y_test, y_probas)\n",
+ "plt.show()\n",
+ "skplt.metrics.plot_cumulative_gain(y_test, y_probas)\n",
+ "plt.show()\n",
+ "\n",
+ "\n",
"xgb.plot_tree(xg_clf,num_trees=0)\n",
"plt.rcParams['figure.figsize'] = [50, 10]\n",
"plt.show()\n",
+ "\n",
"xgb.plot_importance(xg_clf)\n",
"plt.rcParams['figure.figsize'] = [5, 5]\n",
"plt.show()"
diff --git a/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz b/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz
index 461a4ebeb..3edcdce6c 100644
Binary files a/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz and b/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz differ
diff --git a/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf b/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf
index 22edceb81..eb8c19711 100644
Binary files a/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf and b/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf differ
diff --git a/doc/src/DecisionTrees/DecisionTrees.do.txt b/doc/src/DecisionTrees/DecisionTrees.do.txt
index 41ed874f2..2c9ec5a9a 100644
--- a/doc/src/DecisionTrees/DecisionTrees.do.txt
+++ b/doc/src/DecisionTrees/DecisionTrees.do.txt
@@ -1766,7 +1766,7 @@ which leads to
where we have redefined the error as
!bt
\[
-\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(\bm{X}_{i*})}{\sum_{i=0}^{n-1}w_i^m},
+\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i)}{\sum_{i=0}^{n-1}w_i^m},
\]
!et
which leads to an update of
@@ -1809,13 +1809,13 @@ o We start by initializing all weights to $w_i = 1/n$, with $i=0,1,2,\dots n-1$.
o We rewrite the misclassification error as
!bt
\[
-\mathrm{err}=\frac{\sum_{i=0}^{n-1}w_iI(y_i\ne G(x_i})}{\sum_{i=0}^{n-1}w_i},
+\mathrm{\overline{err}}_m=\frac{\sum_{i=0}^{n-1}w_i^m I(y_i\ne G(x_i))}{\sum_{i=0}^{n-1}w_i},
\]
!et
o Then we start looping over all attempts at classifying, namely we start an iterative process for $m=1:M$, where $M$ is the final number of classifications. Our given classifier could for example be a plain decision tree.
o Fit then a given classifier to the training using the weights $w_i$.
o Compute then $\mathrm{err}$ and figure out which events are classified properly and which are classified wrongly.
- o Define a quantity $\alpha_{m} = \log{(1-\mathrm{err})/\mathrm{err}}$
+ o Define a quantity $\alpha_{m} = \log{(1-\mathrm{\overline{err}}_m)/\mathrm{\overline{err}}_m}$
o Set the new weights to $w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(x_i)}$.
o Compute the new classifier $G(x)= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(x_i)$.
@@ -1870,6 +1870,8 @@ In order to understand the method, let us illustrate its basics by
bringing back the essential steps in linear regression, where our cost
function was the least squares function.
+See discussion during lecture November 8.
+
!split
===== Gradient Boosting, algorithm =====
@@ -1883,7 +1885,7 @@ C(\bm{y},\bm{f})=\sum_{i=0}^{n-1}(y_i-f(x_i))^2.
The way we proceed in an iterative fashion is to
o Initialize our estimate $f_0(x)$.
o For $m=1:M$, we
- o compute the negative gradient vector $\bm{u}_m = -\partial C(\bm{y},\bm{f})/\partial \bm{f}(x)$ at $f(x) = f_{m-1}(x);
+ o compute the negative gradient vector $\bm{u}_m = -\partial C(\bm{y},\bm{f})/\partial \bm{f}(x)$ at $f(x) = f_{m-1}(x)$;
o fit the so-called base-learner to the negative gradient $h_m(u_m,x)$;
o update the estimate $f_m(x) = f_{m-1}(x)+\nu h_m(u_m,x)$;
o The final estimate is then $f_M(x) = \sum_{m=1}^M\nu h_m(u_m,x)$.
@@ -1892,91 +1894,96 @@ o The final estimate is then $f_M(x) = \sum_{m=1}^M\nu h_m(u_m,x)$.
!split
-===== Gradient Boosting, Examples =====
+===== Gradient Boosting, Examples of Regression =====
!bc pycod
-np.random.seed(42)
-X = np.random.rand(100, 1) - 0.5
-y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)
-
-from sklearn.tree import DecisionTreeRegressor
-
-tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg1.fit(X, y)
-
-y2 = y - tree_reg1.predict(X)
-tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg2.fit(X, y2)
-
-y3 = y2 - tree_reg2.predict(X)
-tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg3.fit(X, y3)
-
-X_new = np.array([[0.8]])
-y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))
-
-def plot_predictions(regressors, X, y, axes, label=None, style="r-", data_style="b.", data_label=None):
- x1 = np.linspace(axes[0], axes[1], 500)
- y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)
- plt.plot(X[:, 0], y, data_style, label=data_label)
- plt.plot(x1, y_pred, style, linewidth=2, label=label)
- if label or data_label:
- plt.legend(loc="upper center", fontsize=16)
- plt.axis(axes)
-
-plt.figure(figsize=(11,11))
-
-plt.subplot(321)
-plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h_1(x_1)$", style="g-", data_label="Training set")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-plt.title("Residuals and tree predictions", fontsize=16)
-
-plt.subplot(322)
-plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1)$", data_label="Training set")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-plt.title("Ensemble predictions", fontsize=16)
-
-plt.subplot(323)
-plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_2(x_1)$", style="g-", data_style="k+", data_label="Residuals")
-plt.ylabel("$y - h_1(x_1)$", fontsize=16)
-
-plt.subplot(324)
-plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1)$")
-plt.ylabel("$y$", fontsize=16, rotation=0)
-
-plt.subplot(325)
-plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_3(x_1)$", style="g-", data_style="k+")
-plt.ylabel("$y - h_1(x_1) - h_2(x_1)$", fontsize=16)
-plt.xlabel("$x_1$", fontsize=16)
-
-plt.subplot(326)
-plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$")
-plt.xlabel("$x_1$", fontsize=16)
-plt.ylabel("$y$", fontsize=16, rotation=0)
-
-save_fig("gradient_boosting_plot")
-plt.show()
-
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingRegressor
+from sklearn.preprocessing import StandardScaler
+import scikitplot as skplt
+from sklearn.metrics import mean_squared_error
-gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0, random_state=42)
-gbrt.fit(X, y)
+n = 100
+maxdegree = 6
-gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)
-gbrt_slow.fit(X, y)
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
-plt.figure(figsize=(11,4))
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
-plt.subplot(121)
-plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="Ensemble predictions")
-plt.title("learning_rate={}, n_estimators={}".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)
+for degree in range(1,maxdegree):
+ model = GradientBoostingRegressor(max_depth=degree, n_estimators=100, learning_rate=1.0)
+ model.fit(X_train_scaled,y_train)
+ y_pred = model.predict(X_test_scaled)
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
+ variance[degree] = np.mean( np.var(y_pred) )
+ print('Max depth:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
-plt.subplot(122)
-plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
-plt.title("learning_rate={}, n_estimators={}".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)
-
-save_fig("gbrt_learning_rate_plot")
+plt.xlim(1,maxdegree-1)
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
plt.show()
+!ec
+
+!split
+===== Gradient Boosting, Examples of Classification =====
+!bc pycod
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+import scikitplot as skplt
+from sklearn.ensemble import GradientBoostingClassifier
+from sklearn.model_selection import cross_validate
+
+# Load the data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+gd_clf = GradientBoostingClassifier(max_depth=3, n_estimators=100, learning_rate=1.0)
+gd_clf.fit(X_train_scaled, y_train)
+#Cross validation
+accuracy = cross_validate(gd_clf,X_test_scaled,y_test,cv=10)['test_score']
+print(accuracy)
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(gd_clf.score(X_test_scaled,y_test)))
+
+import scikitplot as skplt
+y_pred = gd_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = gd_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
!ec
@@ -2027,8 +2034,8 @@ X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)
for degree in range(maxdegree):
- model = xgb.XGBRegressor(objective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1,
- max_depth = degree, alpha = 10, n_estimators = 10)
+ model = xgb.XGBRegressor(objective ='reg:squarederror', colsaobjective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1,max_depth = degree, alpha = 10, n_estimators = 200)
+
model.fit(X_train_scaled,y_train)
y_pred = model.predict(X_test_scaled)
polydegree[degree] = degree
@@ -2048,14 +2055,12 @@ plt.plot(polydegree, variance, label='Variance')
plt.legend()
plt.show()
-
!ec
-
-
!split
===== Xgboost on the Cancer Data =====
!bc pycod
+
import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split
@@ -2079,10 +2084,28 @@ X_test_scaled = scaler.transform(X_test)
xg_clf = xgb.XGBClassifier()
xg_clf.fit(X_train_scaled,y_train)
+
+y_test = xg_clf.predict(X_test_scaled)
+
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(xg_clf.score(X_test_scaled,y_test)))
+
+import scikitplot as skplt
+y_pred = xg_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = xg_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
+
+
xgb.plot_tree(xg_clf,num_trees=0)
plt.rcParams['figure.figsize'] = [50, 10]
plt.show()
+
xgb.plot_importance(xg_clf)
plt.rcParams['figure.figsize'] = [5, 5]
plt.show()
+
!ec
diff --git a/doc/src/DecisionTrees/Programs/gdclas.py b/doc/src/DecisionTrees/Programs/gdclas.py
new file mode 100644
index 000000000..c7e81b05e
--- /dev/null
+++ b/doc/src/DecisionTrees/Programs/gdclas.py
@@ -0,0 +1,37 @@
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+import scikitplot as skplt
+from sklearn.ensemble import GradientBoostingClassifier
+from sklearn.model_selection import cross_validate
+
+# Load the data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+gd_clf = GradientBoostingClassifier(max_depth=3, n_estimators=100, learning_rate=1.0)
+gd_clf.fit(X_train_scaled, y_train)
+#Cross validation
+accuracy = cross_validate(gd_clf,X_test_scaled,y_test,cv=10)['test_score']
+print(accuracy)
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(gd_clf.score(X_test_scaled,y_test)))
+
+import scikitplot as skplt
+y_pred = gd_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = gd_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
diff --git a/doc/src/DecisionTrees/Programs/gdreg.py b/doc/src/DecisionTrees/Programs/gdreg.py
new file mode 100644
index 000000000..d6c1601c7
--- /dev/null
+++ b/doc/src/DecisionTrees/Programs/gdreg.py
@@ -0,0 +1,48 @@
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.ensemble import GradientBoostingRegressor
+from sklearn.preprocessing import StandardScaler
+import scikitplot as skplt
+from sklearn.metrics import mean_squared_error
+
+n = 1000
+maxdegree = 6
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(1,maxdegree):
+ model = GradientBoostingRegressor(max_depth=degree, n_estimators=100, learning_rate=1.0)
+ model.fit(X_train_scaled,y_train)
+ y_pred = model.predict(X_test_scaled)
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
+ variance[degree] = np.mean( np.var(y_pred) )
+ print('Max depth:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.xlim(1,maxdegree-1)
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
diff --git a/doc/src/DecisionTrees/Programs/xgcancer.py b/doc/src/DecisionTrees/Programs/xgcancer.py
index 55d910b0d..bad114975 100644
--- a/doc/src/DecisionTrees/Programs/xgcancer.py
+++ b/doc/src/DecisionTrees/Programs/xgcancer.py
@@ -19,10 +19,23 @@ scaler.fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)
-xg_clf = xgb.XGBClassifier()
+xg_clf = xgb.XGBClassifier(max_depth = 4, n_estimators = 200)
xg_clf.fit(X_train_scaled,y_train)
-preds = xg_clf.predict(X_test_scaled)
+y_test = xg_clf.predict(X_test_scaled)
+
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(xg_clf.score(X_test_scaled,y_test)))
+
+import scikitplot as skplt
+y_pred = xg_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = xg_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
+
xgb.plot_tree(xg_clf,num_trees=0)
plt.rcParams['figure.figsize'] = [50, 10]