From b2ba05cf10fa5344f4d244e08636daafed02da64 Mon Sep 17 00:00:00 2001 From: mhjensen Date: Thu, 7 Nov 2019 14:00:20 +0100 Subject: [PATCH] added html files --- .../html/._DecisionTrees-bs054.html | 347 ++++++++++++++++++ .../html/._DecisionTrees-bs055.html | 303 +++++++++++++++ .../html/._DecisionTrees-bs056.html | 336 +++++++++++++++++ .../html/._DecisionTrees-bs057.html | 317 ++++++++++++++++ .../html/DecisionTrees-reveal.html | 10 +- .../html/DecisionTrees-solarized.html | 10 +- doc/pub/DecisionTrees/html/DecisionTrees.html | 10 +- .../DecisionTrees/ipynb/DecisionTrees.ipynb | 10 +- .../ipynb/ipynb-DecisionTrees-src.tar.gz | Bin 294061 -> 294061 bytes .../pdf/DecisionTrees-minted.pdf | Bin 526683 -> 526683 bytes doc/src/DecisionTrees/DecisionTrees.do.txt | 10 +- doc/src/DecisionTrees/Programs/xgregressor.py | 8 +- 12 files changed, 1332 insertions(+), 29 deletions(-) create mode 100644 doc/pub/DecisionTrees/html/._DecisionTrees-bs054.html create mode 100644 doc/pub/DecisionTrees/html/._DecisionTrees-bs055.html create mode 100644 doc/pub/DecisionTrees/html/._DecisionTrees-bs056.html create mode 100644 doc/pub/DecisionTrees/html/._DecisionTrees-bs057.html diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs054.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs054.html new file mode 100644 index 000000000..7f4292f42 --- /dev/null +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs054.html @@ -0,0 +1,347 @@ + + + + + + + + +Data Analysis and Machine Learning: From Decision Trees to Forests and all that + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: From Decision Trees to Forests and all that +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Gradient Boots with Early Stopping

+

+ + +

from sklearn.model_selection import train_test_split
+from sklearn.metrics import mean_squared_error
+
+X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=49)
+
+gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=120, random_state=42)
+gbrt.fit(X_train, y_train)
+
+errors = [mean_squared_error(y_val, y_pred)
+          for y_pred in gbrt.staged_predict(X_val)]
+bst_n_estimators = np.argmin(errors) + 1
+
+gbrt_best = GradientBoostingRegressor(max_depth=2,n_estimators=bst_n_estimators, random_state=42)
+gbrt_best.fit(X_train, y_train)
+
+min_error = np.min(errors)
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.plot(errors, "b.-")
+plt.plot([bst_n_estimators, bst_n_estimators], [0, min_error], "k--")
+plt.plot([0, 120], [min_error, min_error], "k--")
+plt.plot(bst_n_estimators, min_error, "ko")
+plt.text(bst_n_estimators, min_error*1.2, "Minimum", ha="center", fontsize=14)
+plt.axis([0, 120, 0, 0.01])
+plt.xlabel("Number of trees")
+plt.title("Validation error", fontsize=14)
+
+plt.subplot(122)
+plot_predictions([gbrt_best], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
+plt.title("Best model (%d trees)" % bst_n_estimators, fontsize=14)
+
+save_fig("early_stopping_gbrt_plot")
+plt.show()
+
+
+gbrt = GradientBoostingRegressor(max_depth=2, warm_start=True, random_state=42)
+
+min_val_error = float("inf")
+error_going_up = 0
+for n_estimators in range(1, 120):
+    gbrt.n_estimators = n_estimators
+    gbrt.fit(X_train, y_train)
+    y_pred = gbrt.predict(X_val)
+    val_error = mean_squared_error(y_val, y_pred)
+    if val_error < min_val_error:
+        min_val_error = val_error
+        error_going_up = 0
+    else:
+        error_going_up += 1
+        if error_going_up == 5:
+            break  # early stopping
+
+
+print(gbrt.n_estimators)
+print("Minimum validation MSE:", min_val_error)
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs055.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs055.html new file mode 100644 index 000000000..e5de741a6 --- /dev/null +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs055.html @@ -0,0 +1,303 @@ + + + + + + + + +Data Analysis and Machine Learning: From Decision Trees to Forests and all that + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: From Decision Trees to Forests and all that +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

XGBoost: Extreme Gradient Boosting

+ +

+XGBoost or Extreme Gradient +Boosting, is an optimized distributed gradient boosting library +designed to be highly efficient, flexible and portable. It implements +machine learning algorithms under the Gradient Boosting +framework. XGBoost provides a parallel tree boosting that solve many +data science problems in a fast and accurate way. See the article by Chen and Guestrin. + +

+The authors design and build a highly scalable end-to-end tree +boosting system. It has a theoretically justified weighted quantile +sketch for efficient proposal calculation. It introduces a novel sparsity-aware algorithm for parallel tree learning and an effective cache-aware block structure for out-of-core tree learning. + +

+It is now the algorithm which wins essentially all ML competitions!!! + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs056.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs056.html new file mode 100644 index 000000000..93ffabf9c --- /dev/null +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs056.html @@ -0,0 +1,336 @@ + + + + + + + + +Data Analysis and Machine Learning: From Decision Trees to Forests and all that + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: From Decision Trees to Forests and all that +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Regression Case

+ +

+ + +

import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+import xgboost as xgb
+from sklearn.preprocessing import StandardScaler
+import scikitplot as skplt
+from sklearn.metrics import mean_squared_error
+
+n = 100
+maxdegree = 6
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+    model =  xgb.XGBRegressor(objective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1,
+                max_depth = degree, alpha = 10, n_estimators = 10)
+    model.fit(X_train_scaled,y_train)
+    y_pred = model.predict(X_test_scaled)
+    polydegree[degree] = degree
+    error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+    bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
+    variance[degree] = np.mean( np.var(y_pred) )
+    print('Max depth:', degree)
+    print('Error:', error[degree])
+    print('Bias^2:', bias[degree])
+    print('Var:', variance[degree])
+    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.xlim(1,maxdegree-1)
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs057.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs057.html new file mode 100644 index 000000000..08bf5350e --- /dev/null +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs057.html @@ -0,0 +1,317 @@ + + + + + + + + +Data Analysis and Machine Learning: From Decision Trees to Forests and all that + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: From Decision Trees to Forests and all that +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Xgboost on the Cancer Data

+

+ + +

import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import  train_test_split 
+from sklearn.datasets import load_breast_cancer
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+import scikitplot as skplt
+import xgboost as xgb
+# Load the data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+xg_clf = xgb.XGBClassifier()
+xg_clf.fit(X_train_scaled,y_train)
+xgb.plot_tree(xg_clf,num_trees=0)
+plt.rcParams['figure.figsize'] = [50, 10]
+plt.show()
+xgb.plot_importance(xg_clf)
+plt.rcParams['figure.figsize'] = [5, 5]
+plt.show()
+
+

+ +

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html b/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html index 8a42b613e..debbea6c2 100644 --- a/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html +++ b/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html @@ -2431,9 +2431,8 @@ It is now the algorithm which wins essentially all ML competitions!!! import scikitplot as skplt from sklearn.metrics import mean_squared_error -n = 40 -n_boostraps = 100 -maxdegree = 8 +n = 100 +maxdegree = 6 # Make data set. x = np.linspace(-3, 3, n).reshape(-1, 1) @@ -2450,8 +2449,8 @@ X_train_scaled = scaler.transform(X_train) X_test_scaled = scaler.transform(X_test) for degree in range(maxdegree): - model = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1, - max_depth = maxdegree, alpha = 10, n_estimators = 10) + model = xgb.XGBRegressor(objective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1, + max_depth = degree, alpha = 10, n_estimators = 10) model.fit(X_train_scaled,y_train) y_pred = model.predict(X_test_scaled) polydegree[degree] = degree @@ -2464,6 +2463,7 @@ X_test_scaled = scaler.transform(X_test) print('Var:', variance[degree]) print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree])) +plt.xlim(1,maxdegree-1) plt.plot(polydegree, error, label='Error') plt.plot(polydegree, bias, label='bias') plt.plot(polydegree, variance, label='Variance') diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html b/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html index e8e3b4ce1..b220f97f4 100644 --- a/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html +++ b/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html @@ -2410,9 +2410,8 @@ It is now the algorithm which wins essentially all ML competitions!!! import scikitplot as skplt from sklearn.metrics import mean_squared_error -n = 40 -n_boostraps = 100 -maxdegree = 8 +n = 100 +maxdegree = 6 # Make data set. x = np.linspace(-3, 3, n).reshape(-1, 1) @@ -2429,8 +2428,8 @@ X_train_scaled = scaler.transform(X_train) X_test_scaled = scaler.transform(X_test) for degree in range(maxdegree): - model = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1, - max_depth = maxdegree, alpha = 10, n_estimators = 10) + model = xgb.XGBRegressor(objective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1, + max_depth = degree, alpha = 10, n_estimators = 10) model.fit(X_train_scaled,y_train) y_pred = model.predict(X_test_scaled) polydegree[degree] = degree @@ -2443,6 +2442,7 @@ X_test_scaled = scaler.transform(X_test) print('Var:', variance[degree]) print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree])) +plt.xlim(1,maxdegree-1) plt.plot(polydegree, error, label='Error') plt.plot(polydegree, bias, label='bias') plt.plot(polydegree, variance, label='Variance') diff --git a/doc/pub/DecisionTrees/html/DecisionTrees.html b/doc/pub/DecisionTrees/html/DecisionTrees.html index 180cfd4e8..98e5b068e 100644 --- a/doc/pub/DecisionTrees/html/DecisionTrees.html +++ b/doc/pub/DecisionTrees/html/DecisionTrees.html @@ -2415,9 +2415,8 @@ It is now the algorithm which wins essentially all ML competitions!!! import scikitplot as skplt from sklearn.metrics import mean_squared_error -n = 40 -n_boostraps = 100 -maxdegree = 8 +n = 100 +maxdegree = 6 # Make data set. x = np.linspace(-3, 3, n).reshape(-1, 1) @@ -2434,8 +2433,8 @@ X_train_scaled = scaler= scaler.transform(X_test) for degree in range(maxdegree): - model = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1, - max_depth = maxdegree, alpha = 10, n_estimators = 10) + model = xgb.XGBRegressor(objective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1, + max_depth = degree, alpha = 10, n_estimators = 10) model.fit(X_train_scaled,y_train) y_pred = model.predict(X_test_scaled) polydegree[degree] = degree @@ -2448,6 +2447,7 @@ X_test_scaled = scalerprint('Var:', variance[degree]) print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree])) +plt.xlim(1,maxdegree-1) plt.plot(polydegree, error, label='Error') plt.plot(polydegree, bias, label='bias') plt.plot(polydegree, variance, label='Variance') diff --git a/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb b/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb index c27d3d1be..6353b4ab0 100644 --- a/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb +++ b/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb @@ -2509,9 +2509,8 @@ "import scikitplot as skplt\n", "from sklearn.metrics import mean_squared_error\n", "\n", - "n = 40\n", - "n_boostraps = 100\n", - "maxdegree = 8\n", + "n = 100\n", + "maxdegree = 6\n", "\n", "# Make data set.\n", "x = np.linspace(-3, 3, n).reshape(-1, 1)\n", @@ -2528,8 +2527,8 @@ "X_test_scaled = scaler.transform(X_test)\n", "\n", "for degree in range(maxdegree):\n", - " model = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1,\n", - " max_depth = maxdegree, alpha = 10, n_estimators = 10)\n", + " model = xgb.XGBRegressor(objective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1,\n", + " max_depth = degree, alpha = 10, n_estimators = 10)\n", " model.fit(X_train_scaled,y_train)\n", " y_pred = model.predict(X_test_scaled)\n", " polydegree[degree] = degree\n", @@ -2542,6 +2541,7 @@ " print('Var:', variance[degree])\n", " print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))\n", "\n", + "plt.xlim(1,maxdegree-1)\n", "plt.plot(polydegree, error, label='Error')\n", "plt.plot(polydegree, bias, label='bias')\n", "plt.plot(polydegree, variance, label='Variance')\n", diff --git a/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz b/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz index 69d0000cf71a1d7a98992d01fb95137ae504e95a..e1351e4b1c57dcdeece9a9ea51ebfb66af4722a7 100644 GIT binary patch delta 29 kcmZ4cQ*iB1K{okr4hBonMz&Tq##T0_RyO9XY%I%b0FcB8g#Z8m delta 29 lcmZ4cQ*iB1K{okr4u-!%jcl!KjIC@;t!&I&*;tm>005p@2}S?_ diff --git a/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf b/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf index 9139b87ab575125589d9ede0669de34817b13ef3..0f6cc40c4fc831e195c0e9eeae3cf7c969d280d1 100644 GIT binary patch delta 116 zcmcaTN#XV+g@zW!7N!>F7M2#)7Pc+y9siguO^l{@|6^AGF}AP!$L`PWY+z<)U}R!x o;%w$>YT;;U;$&uQY-wy}YG~F7M2#)7Pc+y9sigO3=OAu|6^AGF}AP!$L`PW?BZYH48PW?=5-W@7B>Wa?_^Xy9gKY^PvDNXd2wHV$cK07%^+EC2ui diff --git a/doc/src/DecisionTrees/DecisionTrees.do.txt b/doc/src/DecisionTrees/DecisionTrees.do.txt index ac948a01c..59a5a1bf7 100644 --- a/doc/src/DecisionTrees/DecisionTrees.do.txt +++ b/doc/src/DecisionTrees/DecisionTrees.do.txt @@ -2038,9 +2038,8 @@ from sklearn.preprocessing import StandardScaler import scikitplot as skplt from sklearn.metrics import mean_squared_error -n = 40 -n_boostraps = 100 -maxdegree = 8 +n = 100 +maxdegree = 6 # Make data set. x = np.linspace(-3, 3, n).reshape(-1, 1) @@ -2057,8 +2056,8 @@ X_train_scaled = scaler.transform(X_train) X_test_scaled = scaler.transform(X_test) for degree in range(maxdegree): - model = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1, - max_depth = maxdegree, alpha = 10, n_estimators = 10) + model = xgb.XGBRegressor(objective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1, + max_depth = degree, alpha = 10, n_estimators = 10) model.fit(X_train_scaled,y_train) y_pred = model.predict(X_test_scaled) polydegree[degree] = degree @@ -2071,6 +2070,7 @@ for degree in range(maxdegree): print('Var:', variance[degree]) print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree])) +plt.xlim(1,maxdegree-1) plt.plot(polydegree, error, label='Error') plt.plot(polydegree, bias, label='bias') plt.plot(polydegree, variance, label='Variance') diff --git a/doc/src/DecisionTrees/Programs/xgregressor.py b/doc/src/DecisionTrees/Programs/xgregressor.py index 0aaf7be2f..b22184913 100644 --- a/doc/src/DecisionTrees/Programs/xgregressor.py +++ b/doc/src/DecisionTrees/Programs/xgregressor.py @@ -6,8 +6,7 @@ from sklearn.preprocessing import StandardScaler import scikitplot as skplt from sklearn.metrics import mean_squared_error -n = 40 -n_boostraps = 100 +n = 500 maxdegree = 8 # Make data set. @@ -25,8 +24,8 @@ X_train_scaled = scaler.transform(X_train) X_test_scaled = scaler.transform(X_test) for degree in range(maxdegree): - model = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1, - max_depth = maxdegree, alpha = 10, n_estimators = 10) + model = xgb.XGBRegressor(objective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1, + max_depth = degree, alpha = 10, n_estimators = 10) model.fit(X_train_scaled,y_train) y_pred = model.predict(X_test_scaled) polydegree[degree] = degree @@ -39,6 +38,7 @@ for degree in range(maxdegree): print('Var:', variance[degree]) print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree])) +plt.xlim(1,maxdegree-1) plt.plot(polydegree, error, label='Error') plt.plot(polydegree, bias, label='bias') plt.plot(polydegree, variance, label='Variance')