-Random forests provide an improvement over bagged trees by way of a -small tweak that decorrelates the trees. -
-As in bagging, we build a -number of decision trees on bootstrapped training samples. But when -building these decision trees, each time a split in a tree is -considered, a random sample of \( m \) predictors is chosen as split -candidates from the full set of \( p \) predictors. The split is allowed to -use only one of those \( m \) predictors. + +
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
-
-A fresh sample of \( m \) predictors is
-taken at each split, and typically we choose
+n = 100
+n_boostraps = 100
+maxdepth = 8
-$$
-m\approx \sqrt{p}.
-$$
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdepth)
+bias = np.zeros(maxdepth)
+variance = np.zeros(maxdepth)
+polydegree = np.zeros(maxdepth)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
-
-In building a random forest, at
-each split in the tree, the algorithm is not even allowed to consider
-a majority of the available predictors.
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
-
-The reason for this is rather clever. Suppose that there is one very
-strong predictor in the data set, along with a number of other
-moderately strong predictors. Then in the collection of bagged
-variable importance random forest trees, most or all of the trees will
-use this strong predictor in the top split. Consequently, all of the
-bagged trees will look quite similar to each other. Hence the
-predictions from the bagged trees will be highly correlated.
-Unfortunately, averaging many highly correlated quantities does not
-lead to as large of a reduction in variance as averaging many
-uncorrelated quanti- ties. In particular, this means that bagging will
-not lead to a substantial reduction in variance over a single tree in
-this setting.
+for degree in range(1,maxdepth):
+ model = DecisionTreeRegressor(max_depth=degree)
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled)#.ravel()
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.xlim(1,maxdepth)
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
@@ -307,7 +322,7 @@ this setting.
-We will grow of forest of say \( M \) trees. +Random forests provide an improvement over bagged trees by way of a +small tweak that decorrelates the trees. -
+As in bagging, we build a +number of decision trees on bootstrapped training samples. But when +building these decision trees, each time a split in a tree is +considered, a random sample of \( m \) predictors is chosen as split +candidates from the full set of \( p \) predictors. The split is allowed to +use only one of those \( m \) predictors. -
+A fresh sample of \( m \) predictors is +taken at each split, and typically we choose -
+In building a random forest, at +each split in the tree, the algorithm is not even allowed to consider +a majority of the available predictors. -
+The reason for this is rather clever. Suppose that there is one very +strong predictor in the data set, along with a number of other +moderately strong predictors. Then in the collection of bagged +variable importance random forest trees, most or all of the trees will +use this strong predictor in the top split. Consequently, all of the +bagged trees will look quite similar to each other. Hence the +predictions from the bagged trees will be highly correlated. +Unfortunately, averaging many highly correlated quantities does not +lead to as large of a reduction in variance as averaging many +uncorrelated quanti- ties. In particular, this means that bagging will +not lead to a substantial reduction in variance over a single tree in +this setting. +
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs043.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs043.html index c4bc06499..45f547496 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs043.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs043.html @@ -101,37 +101,36 @@ Automatically generated HTML file from DocOnce source '___sec37'), ('Now Bagging', 2, None, '___sec38'), ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), - ('Random forests', 2, None, '___sec40'), - ('Random Forest Algorithm', 2, None, '___sec41'), + ('Changing the Level of the Decision Tree', 2, None, '___sec40'), + ('Random forests', 2, None, '___sec41'), + ('Random Forest Algorithm', 2, None, '___sec42'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec42'), + '___sec43'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec43'), - ('Bootstrap with Random Forests Instead of a Single Tree', + '___sec44'), + ('Bootstrap with Random Forests Instead of a Single Tree, own ' + 'Bagging', 2, None, - '___sec44'), - ("Boosting, a Bird'e Eye", 2, None, '___sec45'), + '___sec45'), + ("Boosting, a Bird'e Eye", 2, None, '___sec46'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec46'), - ('Basic Steps of AdaBoost', 2, None, '___sec47'), - ('Figure to Illustrate the Iterative Classification Process', - 2, - None, - '___sec48'), + '___sec47'), + ('Basic Steps of AdaBoost', 2, None, '___sec48'), ('AdaBoost Examples', 2, None, '___sec49'), ('Gradient boosting: Basics', 2, None, '___sec50'), ('Gradient Boosting, algorithm', 2, None, '___sec51'), ('Gradient Boosting, Examples', 2, None, '___sec52'), ('Gradient Boots with Early Stopping', 2, None, '___sec53'), ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), - ('Xgboost on the Cancer Data', 2, None, '___sec55')]} + ('Regression Case', 2, None, '___sec55'), + ('Xgboost on the Cancer Data', 2, None, '___sec56')]} end of tocinfo --> @@ -209,22 +208,23 @@ MathJax.Hub.Config({
+We will grow of forest of say \( M \) trees. - -
import matplotlib.pyplot as plt
-import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn.datasets import load_breast_cancer
-from sklearn.svm import SVC
-from sklearn.linear_model import LogisticRegression
-from sklearn.tree import DecisionTreeClassifier
+
+- For \( m=1:M \) we
-# Load the data
-cancer = load_breast_cancer()
+
+ - Draw a bootstrap sample of from the training data organized in our \( \boldsymbol{X} \) matrix.
+ - We grow then a random forest tree \( T_m \) based on the bootstrapped data by repeating the steps outlined till we reach the maximum node size is reached
-X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-print(X_train.shape)
-print(X_test.shape)
-# Logistic Regression
-logreg = LogisticRegression(solver='lbfgs')
-logreg.fit(X_train, y_train)
-print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
-# Support vector machine
-svm = SVC(gamma='auto', C=100)
-svm.fit(X_train, y_train)
-print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
-# Decision Trees
-deep_tree_clf = DecisionTreeClassifier(max_depth=None)
-deep_tree_clf.fit(X_train, y_train)
-print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
-#now scale the data
-from sklearn.preprocessing import StandardScaler
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-# Logistic Regression
-logreg.fit(X_train_scaled, y_train)
-print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Support Vector Machine
-svm.fit(X_train_scaled, y_train)
-print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Decision Trees
-deep_tree_clf.fit(X_train_scaled, y_train)
-print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
+
+ - we select \( m \le p \) varibales at random from the \( p \) predictors/features
+ - pick the best split point among the \( m \) features using either the CART algorithm or the ID3 for classification and create a new node
+ - split the node into daughter nodes
+
+
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.preprocessing import LabelEncoder
-from sklearn.model_selection import cross_validate
-# Data set not specificied
-#Instantiate the model with 500 trees and entropy as splitting criteria
-Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy")
-Random_Forest_model.fit(X_train_scaled, y_train)
-#Cross validation
-accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score']
-print(accuracy)
-print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test)))
+- Output then the ensemble of trees \( \{T_m\}_1^{M} \) and make predictions for either a regression type of problem or a classification type of problem.
+
-
-import scikitplot as skplt
-y_pred = Random_Forest_model.predict(X_test_scaled)
-skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
-plt.show()
-y_probas = Random_Forest_model.predict_proba(X_test_scaled)
-skplt.metrics.plot_roc(y_test, y_probas)
-plt.show()
-skplt.metrics.plot_cumulative_gain(y_test, y_probas)
-plt.show()
-
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs044.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs044.html index 2401d68f0..7e7fe255f 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs044.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs044.html @@ -101,37 +101,36 @@ Automatically generated HTML file from DocOnce source '___sec37'), ('Now Bagging', 2, None, '___sec38'), ('Making our own Bagging with Bootstrap', 2, None, '___sec39'), - ('Random forests', 2, None, '___sec40'), - ('Random Forest Algorithm', 2, None, '___sec41'), + ('Changing the Level of the Decision Tree', 2, None, '___sec40'), + ('Random forests', 2, None, '___sec41'), + ('Random Forest Algorithm', 2, None, '___sec42'), ('Random Forests Compared with other Methods on the Cancer Data', 2, None, - '___sec42'), + '___sec43'), ('Compare Bagging on Trees with Random Forests', 2, None, - '___sec43'), - ('Bootstrap with Random Forests Instead of a Single Tree', + '___sec44'), + ('Bootstrap with Random Forests Instead of a Single Tree, own ' + 'Bagging', 2, None, - '___sec44'), - ("Boosting, a Bird'e Eye", 2, None, '___sec45'), + '___sec45'), + ("Boosting, a Bird'e Eye", 2, None, '___sec46'), ('Adaptive boosting: AdaBoost, Basic Algorithm', 2, None, - '___sec46'), - ('Basic Steps of AdaBoost', 2, None, '___sec47'), - ('Figure to Illustrate the Iterative Classification Process', - 2, - None, - '___sec48'), + '___sec47'), + ('Basic Steps of AdaBoost', 2, None, '___sec48'), ('AdaBoost Examples', 2, None, '___sec49'), ('Gradient boosting: Basics', 2, None, '___sec50'), ('Gradient Boosting, algorithm', 2, None, '___sec51'), ('Gradient Boosting, Examples', 2, None, '___sec52'), ('Gradient Boots with Early Stopping', 2, None, '___sec53'), ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'), - ('Xgboost on the Cancer Data', 2, None, '___sec55')]} + ('Regression Case', 2, None, '___sec55'), + ('Xgboost on the Cancer Data', 2, None, '___sec56')]} end of tocinfo --> @@ -209,22 +208,23 @@ MathJax.Hub.Config({
-
bag_clf = BaggingClassifier(
- DecisionTreeClassifier(splitter="random", max_leaf_nodes=16, random_state=42),
- n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1, random_state=42)
-+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+from sklearn.svm import SVC
+from sklearn.linear_model import LogisticRegression
+from sklearn.tree import DecisionTreeClassifier
+
+# Load the data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+# Logistic Regression
+logreg = LogisticRegression(solver='lbfgs')
+logreg.fit(X_train, y_train)
+print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
+# Support vector machine
+svm = SVC(gamma='auto', C=100)
+svm.fit(X_train, y_train)
+print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
+# Decision Trees
+deep_tree_clf = DecisionTreeClassifier(max_depth=None)
+deep_tree_clf.fit(X_train, y_train)
+print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+# Logistic Regression
+logreg.fit(X_train_scaled, y_train)
+print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Support Vector Machine
+svm.fit(X_train_scaled, y_train)
+print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Decision Trees
+deep_tree_clf.fit(X_train_scaled, y_train)
+print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
+
-
-bag_clf.fit(X_train, y_train)
-y_pred = bag_clf.predict(X_test)
from sklearn.ensemble import RandomForestClassifier
-rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)
-rnd_clf.fit(X_train, y_train)
-y_pred_rf = rnd_clf.predict(X_test)
-np.sum(y_pred == y_pred_rf) / len(y_pred)
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+# Data set not specificied
+#Instantiate the model with 500 trees and entropy as splitting criteria
+Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy")
+Random_Forest_model.fit(X_train_scaled, y_train)
+#Cross validation
+accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score']
+print(accuracy)
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test)))
+
+
+import scikitplot as skplt
+y_pred = Random_Forest_model.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = Random_Forest_model.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
@@ -285,7 +336,7 @@ np.sum(y_pred =
53
54
...
- 57
+ 58
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs045.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs045.html
index 1431d4eb5..4372127c9 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs045.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs045.html
@@ -101,37 +101,36 @@ Automatically generated HTML file from DocOnce source
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
- ('Random forests', 2, None, '___sec40'),
- ('Random Forest Algorithm', 2, None, '___sec41'),
+ ('Changing the Level of the Decision Tree', 2, None, '___sec40'),
+ ('Random forests', 2, None, '___sec41'),
+ ('Random Forest Algorithm', 2, None, '___sec42'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec42'),
+ '___sec43'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec43'),
- ('Bootstrap with Random Forests Instead of a Single Tree',
+ '___sec44'),
+ ('Bootstrap with Random Forests Instead of a Single Tree, own '
+ 'Bagging',
2,
None,
- '___sec44'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
+ '___sec45'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec46'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec46'),
- ('Basic Steps of AdaBoost', 2, None, '___sec47'),
- ('Figure to Illustrate the Iterative Classification Process',
- 2,
- None,
- '___sec48'),
+ '___sec47'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec48'),
('AdaBoost Examples', 2, None, '___sec49'),
('Gradient boosting: Basics', 2, None, '___sec50'),
('Gradient Boosting, algorithm', 2, None, '___sec51'),
('Gradient Boosting, Examples', 2, None, '___sec52'),
('Gradient Boots with Early Stopping', 2, None, '___sec53'),
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
- ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
+ ('Regression Case', 2, None, '___sec55'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec56')]}
end of tocinfo -->
@@ -209,22 +208,23 @@ MathJax.Hub.Config({
Please, not the moons again! Voting and Bagging
Now Bagging
Making our own Bagging with Bootstrap
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Bootstrap with Random Forests Instead of a Single Tree
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
+ Changing the Level of the Decision Tree
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree, own Bagging
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
AdaBoost Examples
Gradient boosting: Basics
Gradient Boosting, algorithm
Gradient Boosting, Examples
Gradient Boots with Early Stopping
XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Regression Case
+ Xgboost on the Cancer Data
@@ -240,62 +240,24 @@ MathJax.Hub.Config({
-Bootstrap with Random Forests Instead of a Single Tree
-
+Compare Bagging on Trees with Random Forests
-
import matplotlib.pyplot as plt
-import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn.pipeline import make_pipeline
-from sklearn.utils import resample
-from sklearn.ensemble import RandomForestRegressor
+bag_clf = BaggingClassifier(
+ DecisionTreeClassifier(splitter="random", max_leaf_nodes=16, random_state=42),
+ n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1, random_state=42)
+
+
-np.random.seed(2018)
-
-n = 40
-n_boostraps = 100
-maxdegree = 14
-
-# Make data set.
-x = np.linspace(-3, 3, n).reshape(-1, 1)
-y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
-error = np.zeros(maxdegree)
-bias = np.zeros(maxdegree)
-variance = np.zeros(maxdegree)
-polydegree = np.zeros(maxdegree)
-X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
-
-from sklearn.preprocessing import StandardScaler
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-
-for degree in range(maxdegree):
- model = RandomForestRegressor()
- y_pred = np.empty((y_test.shape[0], n_boostraps))
- for i in range(n_boostraps):
- x_, y_ = resample(X_train_scaled, y_train)
- model.fit(x_, y_)
- y_pred[:, i] = model.predict(X_test_scaled).ravel()
-
- polydegree[degree] = degree
- error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
- bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
- variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
- print('Polynomial degree:', degree)
- print('Error:', error[degree])
- print('Bias^2:', bias[degree])
- print('Var:', variance[degree])
- print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
-
-plt.plot(polydegree, error, label='Error')
-plt.plot(polydegree, bias, label='bias')
-plt.plot(polydegree, variance, label='Variance')
-plt.legend()
-plt.show()
+
+
bag_clf.fit(X_train, y_train)
+y_pred = bag_clf.predict(X_test)
+from sklearn.ensemble import RandomForestClassifier
+rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)
+rnd_clf.fit(X_train, y_train)
+y_pred_rf = rnd_clf.predict(X_test)
+np.sum(y_pred == y_pred_rf) / len(y_pred)
@@ -323,7 +285,7 @@ plt.show()
54
55
...
- 57
+ 58
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs046.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs046.html
index 78f3f81a4..09ecd3191 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs046.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs046.html
@@ -101,37 +101,36 @@ Automatically generated HTML file from DocOnce source
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
- ('Random forests', 2, None, '___sec40'),
- ('Random Forest Algorithm', 2, None, '___sec41'),
+ ('Changing the Level of the Decision Tree', 2, None, '___sec40'),
+ ('Random forests', 2, None, '___sec41'),
+ ('Random Forest Algorithm', 2, None, '___sec42'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec42'),
+ '___sec43'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec43'),
- ('Bootstrap with Random Forests Instead of a Single Tree',
+ '___sec44'),
+ ('Bootstrap with Random Forests Instead of a Single Tree, own '
+ 'Bagging',
2,
None,
- '___sec44'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
+ '___sec45'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec46'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec46'),
- ('Basic Steps of AdaBoost', 2, None, '___sec47'),
- ('Figure to Illustrate the Iterative Classification Process',
- 2,
- None,
- '___sec48'),
+ '___sec47'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec48'),
('AdaBoost Examples', 2, None, '___sec49'),
('Gradient boosting: Basics', 2, None, '___sec50'),
('Gradient Boosting, algorithm', 2, None, '___sec51'),
('Gradient Boosting, Examples', 2, None, '___sec52'),
('Gradient Boots with Early Stopping', 2, None, '___sec53'),
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
- ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
+ ('Regression Case', 2, None, '___sec55'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec56')]}
end of tocinfo -->
@@ -209,22 +208,23 @@ MathJax.Hub.Config({
Please, not the moons again! Voting and Bagging
Now Bagging
Making our own Bagging with Bootstrap
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Bootstrap with Random Forests Instead of a Single Tree
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
+ Changing the Level of the Decision Tree
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree, own Bagging
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
AdaBoost Examples
Gradient boosting: Basics
Gradient Boosting, algorithm
Gradient Boosting, Examples
Gradient Boots with Early Stopping
XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Regression Case
+ Xgboost on the Cancer Data
@@ -240,20 +240,63 @@ MathJax.Hub.Config({
-Boosting, a Bird'e Eye
+Bootstrap with Random Forests Instead of a Single Tree, own Bagging
-The basic idea is to combine weak classifiers in order to create a good
-classifier. With a weak classifier we often intend a classifier which
-produces results which are only slightly better than we would get by
-random guesses.
-
-This is done by applying in an iterative way a weak (or a standard
-classifier like decision trees) to modify the data. In each iteration
-we emphasize those observations which are misclassified by weighting
-them with a factor.
+
+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.ensemble import RandomForestRegressor
+np.random.seed(2018)
+
+n = 100
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = RandomForestRegressor()
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_.ravel())
+ y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
@@ -280,7 +323,7 @@ them with a factor.
55
56
...
- 57
+ 58
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs047.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs047.html
index 6cddc74f8..2c10930ed 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs047.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs047.html
@@ -101,37 +101,36 @@ Automatically generated HTML file from DocOnce source
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
- ('Random forests', 2, None, '___sec40'),
- ('Random Forest Algorithm', 2, None, '___sec41'),
+ ('Changing the Level of the Decision Tree', 2, None, '___sec40'),
+ ('Random forests', 2, None, '___sec41'),
+ ('Random Forest Algorithm', 2, None, '___sec42'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec42'),
+ '___sec43'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec43'),
- ('Bootstrap with Random Forests Instead of a Single Tree',
+ '___sec44'),
+ ('Bootstrap with Random Forests Instead of a Single Tree, own '
+ 'Bagging',
2,
None,
- '___sec44'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
+ '___sec45'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec46'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec46'),
- ('Basic Steps of AdaBoost', 2, None, '___sec47'),
- ('Figure to Illustrate the Iterative Classification Process',
- 2,
- None,
- '___sec48'),
+ '___sec47'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec48'),
('AdaBoost Examples', 2, None, '___sec49'),
('Gradient boosting: Basics', 2, None, '___sec50'),
('Gradient Boosting, algorithm', 2, None, '___sec51'),
('Gradient Boosting, Examples', 2, None, '___sec52'),
('Gradient Boots with Early Stopping', 2, None, '___sec53'),
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
- ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
+ ('Regression Case', 2, None, '___sec55'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec56')]}
end of tocinfo -->
@@ -209,22 +208,23 @@ MathJax.Hub.Config({
Please, not the moons again! Voting and Bagging
Now Bagging
Making our own Bagging with Bootstrap
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Bootstrap with Random Forests Instead of a Single Tree
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
+ Changing the Level of the Decision Tree
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree, own Bagging
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
AdaBoost Examples
Gradient boosting: Basics
Gradient Boosting, algorithm
Gradient Boosting, Examples
Gradient Boots with Early Stopping
XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Regression Case
+ Xgboost on the Cancer Data
@@ -240,24 +240,19 @@ MathJax.Hub.Config({
-Adaptive boosting: AdaBoost, Basic Algorithm
+Boosting, a Bird'e Eye
-The algorithm here is rather straightforward. Assume that our weak
-classifier is a decision tree and we consider a binary set of outputs
-with \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
-observations. Our design matrix is given in terms of the
-feature/predictor vectors
-\( \boldsymbol{X}=[\boldsymbol{x}_0\boldsymbol{x}_1\dots\boldsymbol{x}_{p-1} \). Finally, we define also a
-classifier determined by our data via a function \( G(\boldsymbol{X}) \). This function tells us how well we are able to classify our outputs/targets \( \boldsymbol{y} \).
+The basic idea is to combine weak classifiers in order to create a good
+classifier. With a weak classifier we often intend a classifier which
+produces results which are only slightly better than we would get by
+random guesses.
-We can then define the misclassification error \( \mathrm{err} \) as
-$$
-\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(\boldsymbol{X}_{i*}),
-$$
-
-where the function \( I() \) is one if we misclassify and zero if we classify correctly.
+This is done by applying in an iterative way a weak (or a standard
+classifier like decision trees) to modify the data. In each iteration
+we emphasize those observations which are misclassified by weighting
+them with a factor.
@@ -284,6 +279,8 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
55
56
57
+ ...
+ 58
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs048.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs048.html
index e13cc85c8..633771aec 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs048.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs048.html
@@ -101,37 +101,36 @@ Automatically generated HTML file from DocOnce source
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
- ('Random forests', 2, None, '___sec40'),
- ('Random Forest Algorithm', 2, None, '___sec41'),
+ ('Changing the Level of the Decision Tree', 2, None, '___sec40'),
+ ('Random forests', 2, None, '___sec41'),
+ ('Random Forest Algorithm', 2, None, '___sec42'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec42'),
+ '___sec43'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec43'),
- ('Bootstrap with Random Forests Instead of a Single Tree',
+ '___sec44'),
+ ('Bootstrap with Random Forests Instead of a Single Tree, own '
+ 'Bagging',
2,
None,
- '___sec44'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
+ '___sec45'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec46'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec46'),
- ('Basic Steps of AdaBoost', 2, None, '___sec47'),
- ('Figure to Illustrate the Iterative Classification Process',
- 2,
- None,
- '___sec48'),
+ '___sec47'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec48'),
('AdaBoost Examples', 2, None, '___sec49'),
('Gradient boosting: Basics', 2, None, '___sec50'),
('Gradient Boosting, algorithm', 2, None, '___sec51'),
('Gradient Boosting, Examples', 2, None, '___sec52'),
('Gradient Boots with Early Stopping', 2, None, '___sec53'),
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
- ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
+ ('Regression Case', 2, None, '___sec55'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec56')]}
end of tocinfo -->
@@ -209,22 +208,23 @@ MathJax.Hub.Config({
Please, not the moons again! Voting and Bagging
Now Bagging
Making our own Bagging with Bootstrap
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Bootstrap with Random Forests Instead of a Single Tree
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
+ Changing the Level of the Decision Tree
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree, own Bagging
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
AdaBoost Examples
Gradient boosting: Basics
Gradient Boosting, algorithm
Gradient Boosting, Examples
Gradient Boots with Early Stopping
XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Regression Case
+ Xgboost on the Cancer Data
@@ -240,42 +240,24 @@ MathJax.Hub.Config({
-Basic Steps of AdaBoost
+Adaptive boosting: AdaBoost, Basic Algorithm
-With the above definitions we are now ready to set up the algorithm for AdaBoost.
-The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases.
-
-
-- We start by initializing all weights to \( w_i = 1/n \), with \( i=0,1,2,\dots n-1 \). It is to see then that \( \sum_{i=0}^{n-1}w_i = 1 \).
-- We rewrite the misclassification error as
-
+The algorithm here is rather straightforward. Assume that our weak
+classifier is a decision tree and we consider a binary set of outputs
+with \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
+observations. Our design matrix is given in terms of the
+feature/predictor vectors
+\( \boldsymbol{X}=[\boldsymbol{x}_0\boldsymbol{x}_1\dots\boldsymbol{x}_{p-1} \). Finally, we define also a
+classifier determined by our data via a function \( G(\boldsymbol{X}) \). This function tells us how well we are able to classify our outputs/targets \( \boldsymbol{y} \).
+
+We can then define the misclassification error \( \mathrm{err} \) as
$$
-\mathrm{err}=\frac{\sum_{i=0}^{n-1}w_iI(y_i\ne G(\boldsymbol{X}_{i*})}{\sum_{i=0}^{n-1}w_i},
+\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(\boldsymbol{X}_{i*}),
$$
-
-
-- Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree.
-
-
- - Fit thus a given classifier to the training using the weights \( w_i \).
- - Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.
- - Define a quantity $\alpha_{m} = \log{(1-\mathrm{err})/\mathrm{err}}
- - Set the new weights to $w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(\boldsymbol{X}_{i*})}.
-
-
- Compute the new classifier $G(\boldsymbol{X})= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(\boldsymbol{X}_{i*}).
-
-
-For the iterations with \( m \le 2 \) the weights are modified
-individually at each steps. The obersvations which were misclassified
-at iteration \( m-1 \) have a weight which is larger than those which were
-classified properly. As this proceeds, the observations which were
-difficult to classifiy correctly are given a larger influence. Each
-new classification step \( m \) is then forced to concentrate on those
-observations that are missed in the previous iterations.
+where the function \( I() \) is one if we misclassify and zero if we classify correctly.
@@ -301,6 +283,7 @@ observations that are missed in the previous iterations.
55
56
57
+ 58
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs049.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs049.html
index 4a6ad8a24..0a56c63ac 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs049.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs049.html
@@ -101,37 +101,36 @@ Automatically generated HTML file from DocOnce source
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
- ('Random forests', 2, None, '___sec40'),
- ('Random Forest Algorithm', 2, None, '___sec41'),
+ ('Changing the Level of the Decision Tree', 2, None, '___sec40'),
+ ('Random forests', 2, None, '___sec41'),
+ ('Random Forest Algorithm', 2, None, '___sec42'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec42'),
+ '___sec43'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec43'),
- ('Bootstrap with Random Forests Instead of a Single Tree',
+ '___sec44'),
+ ('Bootstrap with Random Forests Instead of a Single Tree, own '
+ 'Bagging',
2,
None,
- '___sec44'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
+ '___sec45'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec46'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec46'),
- ('Basic Steps of AdaBoost', 2, None, '___sec47'),
- ('Figure to Illustrate the Iterative Classification Process',
- 2,
- None,
- '___sec48'),
+ '___sec47'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec48'),
('AdaBoost Examples', 2, None, '___sec49'),
('Gradient boosting: Basics', 2, None, '___sec50'),
('Gradient Boosting, algorithm', 2, None, '___sec51'),
('Gradient Boosting, Examples', 2, None, '___sec52'),
('Gradient Boots with Early Stopping', 2, None, '___sec53'),
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
- ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
+ ('Regression Case', 2, None, '___sec55'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec56')]}
end of tocinfo -->
@@ -209,22 +208,23 @@ MathJax.Hub.Config({
Please, not the moons again! Voting and Bagging
Now Bagging
Making our own Bagging with Bootstrap
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Bootstrap with Random Forests Instead of a Single Tree
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
+ Changing the Level of the Decision Tree
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree, own Bagging
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
AdaBoost Examples
Gradient boosting: Basics
Gradient Boosting, algorithm
Gradient Boosting, Examples
Gradient Boots with Early Stopping
XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Regression Case
+ Xgboost on the Cancer Data
@@ -240,7 +240,42 @@ MathJax.Hub.Config({
-Figure to Illustrate the Iterative Classification Process
+Basic Steps of AdaBoost
+
+
+With the above definitions we are now ready to set up the algorithm for AdaBoost.
+The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases.
+
+
+- We start by initializing all weights to \( w_i = 1/n \), with \( i=0,1,2,\dots n-1 \). It is to see then that \( \sum_{i=0}^{n-1}w_i = 1 \).
+- We rewrite the misclassification error as
+
+
+$$
+\mathrm{err}=\frac{\sum_{i=0}^{n-1}w_iI(y_i\ne G(\boldsymbol{X}_{i*})}{\sum_{i=0}^{n-1}w_i},
+$$
+
+
+
+- Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree.
+
+
+ - Fit then a given classifier to the training using the weights \( w_i \).
+ - Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.
+ - Define a quantity $\alpha_{m} = \log{(1-\mathrm{err})/\mathrm{err}}
+ - Set the new weights to $w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(\boldsymbol{X}_{i*})}.
+
+
+ Compute the new classifier $G(\boldsymbol{X})= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(\boldsymbol{X}_{i*}).
+
+
+For the iterations with \( m \le 2 \) the weights are modified
+individually at each steps. The obersvations which were misclassified
+at iteration \( m-1 \) have a weight which is larger than those which were
+classified properly. As this proceeds, the observations which were
+difficult to classifiy correctly are given a larger influence. Each
+new classification step \( m \) is then forced to concentrate on those
+observations that are missed in the previous iterations.
@@ -265,6 +300,7 @@ MathJax.Hub.Config({
55
56
57
+ 58
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs050.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs050.html
index 4d7d6c3fa..c12d3cb33 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs050.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs050.html
@@ -101,37 +101,36 @@ Automatically generated HTML file from DocOnce source
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
- ('Random forests', 2, None, '___sec40'),
- ('Random Forest Algorithm', 2, None, '___sec41'),
+ ('Changing the Level of the Decision Tree', 2, None, '___sec40'),
+ ('Random forests', 2, None, '___sec41'),
+ ('Random Forest Algorithm', 2, None, '___sec42'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec42'),
+ '___sec43'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec43'),
- ('Bootstrap with Random Forests Instead of a Single Tree',
+ '___sec44'),
+ ('Bootstrap with Random Forests Instead of a Single Tree, own '
+ 'Bagging',
2,
None,
- '___sec44'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
+ '___sec45'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec46'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec46'),
- ('Basic Steps of AdaBoost', 2, None, '___sec47'),
- ('Figure to Illustrate the Iterative Classification Process',
- 2,
- None,
- '___sec48'),
+ '___sec47'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec48'),
('AdaBoost Examples', 2, None, '___sec49'),
('Gradient boosting: Basics', 2, None, '___sec50'),
('Gradient Boosting, algorithm', 2, None, '___sec51'),
('Gradient Boosting, Examples', 2, None, '___sec52'),
('Gradient Boots with Early Stopping', 2, None, '___sec53'),
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
- ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
+ ('Regression Case', 2, None, '___sec55'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec56')]}
end of tocinfo -->
@@ -209,22 +208,23 @@ MathJax.Hub.Config({
Please, not the moons again! Voting and Bagging
Now Bagging
Making our own Bagging with Bootstrap
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Bootstrap with Random Forests Instead of a Single Tree
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
+ Changing the Level of the Decision Tree
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree, own Bagging
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
AdaBoost Examples
Gradient boosting: Basics
Gradient Boosting, algorithm
Gradient Boosting, Examples
Gradient Boots with Early Stopping
XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Regression Case
+ Xgboost on the Cancer Data
@@ -302,6 +302,7 @@ plt.show()
55
56
57
+ 58
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs051.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs051.html
index fe4dd40f5..cbbe511d6 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs051.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs051.html
@@ -101,37 +101,36 @@ Automatically generated HTML file from DocOnce source
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
- ('Random forests', 2, None, '___sec40'),
- ('Random Forest Algorithm', 2, None, '___sec41'),
+ ('Changing the Level of the Decision Tree', 2, None, '___sec40'),
+ ('Random forests', 2, None, '___sec41'),
+ ('Random Forest Algorithm', 2, None, '___sec42'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec42'),
+ '___sec43'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec43'),
- ('Bootstrap with Random Forests Instead of a Single Tree',
+ '___sec44'),
+ ('Bootstrap with Random Forests Instead of a Single Tree, own '
+ 'Bagging',
2,
None,
- '___sec44'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
+ '___sec45'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec46'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec46'),
- ('Basic Steps of AdaBoost', 2, None, '___sec47'),
- ('Figure to Illustrate the Iterative Classification Process',
- 2,
- None,
- '___sec48'),
+ '___sec47'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec48'),
('AdaBoost Examples', 2, None, '___sec49'),
('Gradient boosting: Basics', 2, None, '___sec50'),
('Gradient Boosting, algorithm', 2, None, '___sec51'),
('Gradient Boosting, Examples', 2, None, '___sec52'),
('Gradient Boots with Early Stopping', 2, None, '___sec53'),
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
- ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
+ ('Regression Case', 2, None, '___sec55'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec56')]}
end of tocinfo -->
@@ -209,22 +208,23 @@ MathJax.Hub.Config({
Please, not the moons again! Voting and Bagging
Now Bagging
Making our own Bagging with Bootstrap
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Bootstrap with Random Forests Instead of a Single Tree
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
+ Changing the Level of the Decision Tree
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree, own Bagging
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
AdaBoost Examples
Gradient boosting: Basics
Gradient Boosting, algorithm
Gradient Boosting, Examples
Gradient Boots with Early Stopping
XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Regression Case
+ Xgboost on the Cancer Data
@@ -273,6 +273,7 @@ function was the least squares function.
55
56
57
+ 58
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs052.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs052.html
index 0f3729124..5b430bec9 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs052.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs052.html
@@ -101,37 +101,36 @@ Automatically generated HTML file from DocOnce source
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
- ('Random forests', 2, None, '___sec40'),
- ('Random Forest Algorithm', 2, None, '___sec41'),
+ ('Changing the Level of the Decision Tree', 2, None, '___sec40'),
+ ('Random forests', 2, None, '___sec41'),
+ ('Random Forest Algorithm', 2, None, '___sec42'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec42'),
+ '___sec43'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec43'),
- ('Bootstrap with Random Forests Instead of a Single Tree',
+ '___sec44'),
+ ('Bootstrap with Random Forests Instead of a Single Tree, own '
+ 'Bagging',
2,
None,
- '___sec44'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
+ '___sec45'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec46'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec46'),
- ('Basic Steps of AdaBoost', 2, None, '___sec47'),
- ('Figure to Illustrate the Iterative Classification Process',
- 2,
- None,
- '___sec48'),
+ '___sec47'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec48'),
('AdaBoost Examples', 2, None, '___sec49'),
('Gradient boosting: Basics', 2, None, '___sec50'),
('Gradient Boosting, algorithm', 2, None, '___sec51'),
('Gradient Boosting, Examples', 2, None, '___sec52'),
('Gradient Boots with Early Stopping', 2, None, '___sec53'),
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
- ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
+ ('Regression Case', 2, None, '___sec55'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec56')]}
end of tocinfo -->
@@ -209,22 +208,23 @@ MathJax.Hub.Config({
Please, not the moons again! Voting and Bagging
Now Bagging
Making our own Bagging with Bootstrap
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Bootstrap with Random Forests Instead of a Single Tree
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
+ Changing the Level of the Decision Tree
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree, own Bagging
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
AdaBoost Examples
Gradient boosting: Basics
Gradient Boosting, algorithm
Gradient Boosting, Examples
Gradient Boots with Early Stopping
XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Regression Case
+ Xgboost on the Cancer Data
@@ -283,6 +283,7 @@ The way we proceed in an iterative fashion is to
55
56
57
+ 58
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs053.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs053.html
index 591f2cd60..6dabddbf2 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs053.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs053.html
@@ -101,37 +101,36 @@ Automatically generated HTML file from DocOnce source
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
- ('Random forests', 2, None, '___sec40'),
- ('Random Forest Algorithm', 2, None, '___sec41'),
+ ('Changing the Level of the Decision Tree', 2, None, '___sec40'),
+ ('Random forests', 2, None, '___sec41'),
+ ('Random Forest Algorithm', 2, None, '___sec42'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec42'),
+ '___sec43'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec43'),
- ('Bootstrap with Random Forests Instead of a Single Tree',
+ '___sec44'),
+ ('Bootstrap with Random Forests Instead of a Single Tree, own '
+ 'Bagging',
2,
None,
- '___sec44'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
+ '___sec45'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec46'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec46'),
- ('Basic Steps of AdaBoost', 2, None, '___sec47'),
- ('Figure to Illustrate the Iterative Classification Process',
- 2,
- None,
- '___sec48'),
+ '___sec47'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec48'),
('AdaBoost Examples', 2, None, '___sec49'),
('Gradient boosting: Basics', 2, None, '___sec50'),
('Gradient Boosting, algorithm', 2, None, '___sec51'),
('Gradient Boosting, Examples', 2, None, '___sec52'),
('Gradient Boots with Early Stopping', 2, None, '___sec53'),
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
- ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
+ ('Regression Case', 2, None, '___sec55'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec56')]}
end of tocinfo -->
@@ -209,22 +208,23 @@ MathJax.Hub.Config({
Please, not the moons again! Voting and Bagging
Now Bagging
Making our own Bagging with Bootstrap
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Bootstrap with Random Forests Instead of a Single Tree
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
+ Changing the Level of the Decision Tree
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree, own Bagging
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
AdaBoost Examples
Gradient boosting: Basics
Gradient Boosting, algorithm
Gradient Boosting, Examples
Gradient Boots with Early Stopping
XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Regression Case
+ Xgboost on the Cancer Data
@@ -346,6 +346,7 @@ plt.show()
55
56
57
+ 58
»
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-bs.html b/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
index a15bd51fb..cfc8c9a63 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
@@ -101,37 +101,36 @@ Automatically generated HTML file from DocOnce source
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
- ('Random forests', 2, None, '___sec40'),
- ('Random Forest Algorithm', 2, None, '___sec41'),
+ ('Changing the Level of the Decision Tree', 2, None, '___sec40'),
+ ('Random forests', 2, None, '___sec41'),
+ ('Random Forest Algorithm', 2, None, '___sec42'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec42'),
+ '___sec43'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec43'),
- ('Bootstrap with Random Forests Instead of a Single Tree',
+ '___sec44'),
+ ('Bootstrap with Random Forests Instead of a Single Tree, own '
+ 'Bagging',
2,
None,
- '___sec44'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
+ '___sec45'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec46'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec46'),
- ('Basic Steps of AdaBoost', 2, None, '___sec47'),
- ('Figure to Illustrate the Iterative Classification Process',
- 2,
- None,
- '___sec48'),
+ '___sec47'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec48'),
('AdaBoost Examples', 2, None, '___sec49'),
('Gradient boosting: Basics', 2, None, '___sec50'),
('Gradient Boosting, algorithm', 2, None, '___sec51'),
('Gradient Boosting, Examples', 2, None, '___sec52'),
('Gradient Boots with Early Stopping', 2, None, '___sec53'),
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
- ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
+ ('Regression Case', 2, None, '___sec55'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec56')]}
end of tocinfo -->
@@ -209,22 +208,23 @@ MathJax.Hub.Config({
Please, not the moons again! Voting and Bagging
Now Bagging
Making our own Bagging with Bootstrap
- Random forests
- Random Forest Algorithm
- Random Forests Compared with other Methods on the Cancer Data
- Compare Bagging on Trees with Random Forests
- Bootstrap with Random Forests Instead of a Single Tree
- Boosting, a Bird'e Eye
- Adaptive boosting: AdaBoost, Basic Algorithm
- Basic Steps of AdaBoost
- Figure to Illustrate the Iterative Classification Process
+ Changing the Level of the Decision Tree
+ Random forests
+ Random Forest Algorithm
+ Random Forests Compared with other Methods on the Cancer Data
+ Compare Bagging on Trees with Random Forests
+ Bootstrap with Random Forests Instead of a Single Tree, own Bagging
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ Basic Steps of AdaBoost
AdaBoost Examples
Gradient boosting: Basics
Gradient Boosting, algorithm
Gradient Boosting, Examples
Gradient Boots with Early Stopping
XGBoost: Extreme Gradient Boosting
- Xgboost on the Cancer Data
+ Regression Case
+ Xgboost on the Cancer Data
@@ -283,7 +283,7 @@ MathJax.Hub.Config({
9
10
...
- 57
+ 58
»
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html b/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
index 496aefa10..8a42b613e 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
@@ -1777,7 +1777,67 @@ plt.show()
-Random forests
+Changing the Level of the Decision Tree
+
+
+
+
+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
+
+n = 100
+n_boostraps = 100
+maxdepth = 8
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdepth)
+bias = np.zeros(maxdepth)
+variance = np.zeros(maxdepth)
+polydegree = np.zeros(maxdepth)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(1,maxdepth):
+ model = DecisionTreeRegressor(max_depth=degree)
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled)#.ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.xlim(1,maxdepth)
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
+
+
+Random forests
Random forests provide an improvement over bagged trees by way of a
@@ -1823,7 +1883,7 @@ this setting.
-Random Forest Algorithm
+Random Forest Algorithm
The algorithm described here can be applied to both classification and regression problems.
@@ -1854,7 +1914,7 @@ We will grow of forest of say \( M \) trees.
-Random Forests Compared with other Methods on the Cancer Data
+Random Forests Compared with other Methods on the Cancer Data
@@ -1928,7 +1988,7 @@ plt.show()
-Compare Bagging on Trees with Random Forests
+Compare Bagging on Trees with Random Forests
@@ -1951,7 +2011,7 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
-Bootstrap with Random Forests Instead of a Single Tree
+Bootstrap with Random Forests Instead of a Single Tree, own Bagging
@@ -1965,7 +2025,7 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
np.random.seed(2018)
-n = 40
+n = 100
n_boostraps = 100
maxdegree = 14
@@ -1989,7 +2049,7 @@ X_test_scaled = scaler.transform(X_test)
y_pred = np.empty((y_test.shape[0], n_boostraps))
for i in range(n_boostraps):
x_, y_ = resample(X_train_scaled, y_train)
- model.fit(x_, y_)
+ model.fit(x_, y_.ravel())
y_pred[:, i] = model.predict(X_test_scaled).ravel()
polydegree[degree] = degree
@@ -2012,7 +2072,7 @@ plt.show()
-Boosting, a Bird'e Eye
+Boosting, a Bird'e Eye
The basic idea is to combine weak classifiers in order to create a good
@@ -2029,7 +2089,7 @@ them with a factor.
-Adaptive boosting: AdaBoost, Basic Algorithm
+Adaptive boosting: AdaBoost, Basic Algorithm
The algorithm here is rather straightforward. Assume that our weak
@@ -2053,7 +2113,7 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
-Basic Steps of AdaBoost
+Basic Steps of AdaBoost
With the above definitions we are now ready to set up the algorithm for AdaBoost.
@@ -2074,7 +2134,7 @@ $$
Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree.
- Fit thus a given classifier to the training using the weights \( w_i \).
+ Fit then a given classifier to the training using the weights \( w_i \).
Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.
Define a quantity $\alpha_{m} = \log{(1-\mathrm{err})/\mathrm{err}}
Set the new weights to $w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(\boldsymbol{X}_{i*})}.
@@ -2093,11 +2153,6 @@ observations that are missed in the previous iterations.
-
-Figure to Illustrate the Iterative Classification Process
-
-
-
AdaBoost Examples
@@ -2363,7 +2418,63 @@ It is now the algorithm which wins essentially all ML competitions!!!
-Xgboost on the Cancer Data
+Regression Case
+
+
+
+
+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+import xgboost as xgb
+from sklearn.preprocessing import StandardScaler
+import scikitplot as skplt
+from sklearn.metrics import mean_squared_error
+
+n = 40
+n_boostraps = 100
+maxdegree = 8
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1,
+ max_depth = maxdegree, alpha = 10, n_estimators = 10)
+ model.fit(X_train_scaled,y_train)
+ y_pred = model.predict(X_test_scaled)
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
+ variance[degree] = np.mean( np.var(y_pred) )
+ print('Max depth:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
+
+
+Xgboost on the Cancer Data
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html b/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html
index b2c1e8ea1..e8e3b4ce1 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html
@@ -121,37 +121,36 @@ div { text-align: justify; text-justify: inter-word; }
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
- ('Random forests', 2, None, '___sec40'),
- ('Random Forest Algorithm', 2, None, '___sec41'),
+ ('Changing the Level of the Decision Tree', 2, None, '___sec40'),
+ ('Random forests', 2, None, '___sec41'),
+ ('Random Forest Algorithm', 2, None, '___sec42'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec42'),
+ '___sec43'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec43'),
- ('Bootstrap with Random Forests Instead of a Single Tree',
+ '___sec44'),
+ ('Bootstrap with Random Forests Instead of a Single Tree, own '
+ 'Bagging',
2,
None,
- '___sec44'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
+ '___sec45'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec46'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec46'),
- ('Basic Steps of AdaBoost', 2, None, '___sec47'),
- ('Figure to Illustrate the Iterative Classification Process',
- 2,
- None,
- '___sec48'),
+ '___sec47'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec48'),
('AdaBoost Examples', 2, None, '___sec49'),
('Gradient boosting: Basics', 2, None, '___sec50'),
('Gradient Boosting, algorithm', 2, None, '___sec51'),
('Gradient Boosting, Examples', 2, None, '___sec52'),
('Gradient Boots with Early Stopping', 2, None, '___sec53'),
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
- ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
+ ('Regression Case', 2, None, '___sec55'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec56')]}
end of tocinfo -->
@@ -1776,7 +1775,66 @@ plt.show()
-
Random forests
+Changing the Level of the Decision Tree
+
+
+
+
+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
+
+n = 100
+n_boostraps = 100
+maxdepth = 8
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdepth)
+bias = np.zeros(maxdepth)
+variance = np.zeros(maxdepth)
+polydegree = np.zeros(maxdepth)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(1,maxdepth):
+ model = DecisionTreeRegressor(max_depth=degree)
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled)#.ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.xlim(1,maxdepth)
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
+
+
Random forests
Random forests provide an improvement over bagged trees by way of a
@@ -1820,7 +1878,7 @@ this setting.
-
Random Forest Algorithm
+Random Forest Algorithm
The algorithm described here can be applied to both classification and regression problems.
@@ -1846,7 +1904,7 @@ We will grow of forest of say \( M \) trees.
-
Random Forests Compared with other Methods on the Cancer Data
+Random Forests Compared with other Methods on the Cancer Data
@@ -1919,7 +1977,7 @@ plt.show()
-
Compare Bagging on Trees with Random Forests
+Compare Bagging on Trees with Random Forests
@@ -1941,7 +1999,7 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
-
Bootstrap with Random Forests Instead of a Single Tree
+Bootstrap with Random Forests Instead of a Single Tree, own Bagging
@@ -1955,7 +2013,7 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
np.random.seed(2018)
-n = 40
+n = 100
n_boostraps = 100
maxdegree = 14
@@ -1979,7 +2037,7 @@ X_test_scaled = scaler.transform(X_test)
y_pred = np.empty((y_test.shape[0], n_boostraps))
for i in range(n_boostraps):
x_, y_ = resample(X_train_scaled, y_train)
- model.fit(x_, y_)
+ model.fit(x_, y_.ravel())
y_pred[:, i] = model.predict(X_test_scaled).ravel()
polydegree[degree] = degree
@@ -2001,7 +2059,7 @@ plt.show()
-
Boosting, a Bird'e Eye
+Boosting, a Bird'e Eye
The basic idea is to combine weak classifiers in order to create a good
@@ -2018,7 +2076,7 @@ them with a factor.
-
Adaptive boosting: AdaBoost, Basic Algorithm
+Adaptive boosting: AdaBoost, Basic Algorithm
The algorithm here is rather straightforward. Assume that our weak
@@ -2040,7 +2098,7 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
-
Basic Steps of AdaBoost
+Basic Steps of AdaBoost
With the above definitions we are now ready to set up the algorithm for AdaBoost.
@@ -2060,7 +2118,7 @@ $$
Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree.
- Fit thus a given classifier to the training using the weights \( w_i \).
+ Fit then a given classifier to the training using the weights \( w_i \).
Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.
Define a quantity $\alpha_{m} = \log{(1-\mathrm{err})/\mathrm{err}}
Set the new weights to $w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(\boldsymbol{X}_{i*})}.
@@ -2080,11 +2138,6 @@ observations that are missed in the previous iterations.
-
Figure to Illustrate the Iterative Classification Process
-
-
-
-
AdaBoost Examples
@@ -2344,7 +2397,62 @@ It is now the algorithm which wins essentially all ML competitions!!!
-
Xgboost on the Cancer Data
+Regression Case
+
+
+
+
+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+import xgboost as xgb
+from sklearn.preprocessing import StandardScaler
+import scikitplot as skplt
+from sklearn.metrics import mean_squared_error
+
+n = 40
+n_boostraps = 100
+maxdegree = 8
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1,
+ max_depth = maxdegree, alpha = 10, n_estimators = 10)
+ model.fit(X_train_scaled,y_train)
+ y_pred = model.predict(X_test_scaled)
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
+ variance[degree] = np.mean( np.var(y_pred) )
+ print('Max depth:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
+
+
Xgboost on the Cancer Data
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees.html b/doc/pub/DecisionTrees/html/DecisionTrees.html
index ede4f121d..180cfd4e8 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees.html
@@ -126,37 +126,36 @@ div { text-align: justify; text-justify: inter-word; }
'___sec37'),
('Now Bagging', 2, None, '___sec38'),
('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
- ('Random forests', 2, None, '___sec40'),
- ('Random Forest Algorithm', 2, None, '___sec41'),
+ ('Changing the Level of the Decision Tree', 2, None, '___sec40'),
+ ('Random forests', 2, None, '___sec41'),
+ ('Random Forest Algorithm', 2, None, '___sec42'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
- '___sec42'),
+ '___sec43'),
('Compare Bagging on Trees with Random Forests',
2,
None,
- '___sec43'),
- ('Bootstrap with Random Forests Instead of a Single Tree',
+ '___sec44'),
+ ('Bootstrap with Random Forests Instead of a Single Tree, own '
+ 'Bagging',
2,
None,
- '___sec44'),
- ("Boosting, a Bird'e Eye", 2, None, '___sec45'),
+ '___sec45'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec46'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
- '___sec46'),
- ('Basic Steps of AdaBoost', 2, None, '___sec47'),
- ('Figure to Illustrate the Iterative Classification Process',
- 2,
- None,
- '___sec48'),
+ '___sec47'),
+ ('Basic Steps of AdaBoost', 2, None, '___sec48'),
('AdaBoost Examples', 2, None, '___sec49'),
('Gradient boosting: Basics', 2, None, '___sec50'),
('Gradient Boosting, algorithm', 2, None, '___sec51'),
('Gradient Boosting, Examples', 2, None, '___sec52'),
('Gradient Boots with Early Stopping', 2, None, '___sec53'),
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
- ('Xgboost on the Cancer Data', 2, None, '___sec55')]}
+ ('Regression Case', 2, None, '___sec55'),
+ ('Xgboost on the Cancer Data', 2, None, '___sec56')]}
end of tocinfo -->
@@ -1781,7 +1780,66 @@ plt.show()
-
Random forests
+Changing the Level of the Decision Tree
+
+
+
+
+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
+
+n = 100
+n_boostraps = 100
+maxdepth = 8
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdepth)
+bias = np.zeros(maxdepth)
+variance = np.zeros(maxdepth)
+polydegree = np.zeros(maxdepth)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(1,maxdepth):
+ model = DecisionTreeRegressor(max_depth=degree)
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled)#.ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.xlim(1,maxdepth)
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
+
+
Random forests
Random forests provide an improvement over bagged trees by way of a
@@ -1825,7 +1883,7 @@ this setting.
-
Random Forest Algorithm
+Random Forest Algorithm
The algorithm described here can be applied to both classification and regression problems.
@@ -1851,7 +1909,7 @@ We will grow of forest of say \( M \) trees.
-
Random Forests Compared with other Methods on the Cancer Data
+Random Forests Compared with other Methods on the Cancer Data
@@ -1924,7 +1982,7 @@ plt.show()
-
Compare Bagging on Trees with Random Forests
+Compare Bagging on Trees with Random Forests
@@ -1946,7 +2004,7 @@ np.sum(y_pred =
-
Bootstrap with Random Forests Instead of a Single Tree
+Bootstrap with Random Forests Instead of a Single Tree, own Bagging
@@ -1960,7 +2018,7 @@ np.sum(y_pred =
np.random.seed(2018)
-n = 40
+n = 100
n_boostraps = 100
maxdegree = 14
@@ -1984,7 +2042,7 @@ X_test_scaled = scaler= np.empty((y_test.shape[0], n_boostraps))
for i in range(n_boostraps):
x_, y_ = resample(X_train_scaled, y_train)
- model.fit(x_, y_)
+ model.fit(x_, y_.ravel())
y_pred[:, i] = model.predict(X_test_scaled).ravel()
polydegree[degree] = degree
@@ -2006,7 +2064,7 @@ plt.show()
-
Boosting, a Bird'e Eye
+Boosting, a Bird'e Eye
The basic idea is to combine weak classifiers in order to create a good
@@ -2023,7 +2081,7 @@ them with a factor.
-
Adaptive boosting: AdaBoost, Basic Algorithm
+Adaptive boosting: AdaBoost, Basic Algorithm
The algorithm here is rather straightforward. Assume that our weak
@@ -2045,7 +2103,7 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
-
Basic Steps of AdaBoost
+Basic Steps of AdaBoost
With the above definitions we are now ready to set up the algorithm for AdaBoost.
@@ -2065,7 +2123,7 @@ $$
Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree.
- Fit thus a given classifier to the training using the weights \( w_i \).
+ Fit then a given classifier to the training using the weights \( w_i \).
Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.
Define a quantity $\alpha_{m} = \log{(1-\mathrm{err})/\mathrm{err}}
Set the new weights to $w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(\boldsymbol{X}_{i*})}.
@@ -2085,11 +2143,6 @@ observations that are missed in the previous iterations.
-
Figure to Illustrate the Iterative Classification Process
-
-
-
-
AdaBoost Examples
@@ -2349,7 +2402,62 @@ It is now the algorithm which wins essentially all ML competitions!!!
-
Xgboost on the Cancer Data
+Regression Case
+
+
+
+
+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+import xgboost as xgb
+from sklearn.preprocessing import StandardScaler
+import scikitplot as skplt
+from sklearn.metrics import mean_squared_error
+
+n = 40
+n_boostraps = 100
+maxdegree = 8
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1,
+ max_depth = maxdegree, alpha = 10, n_estimators = 10)
+ model.fit(X_train_scaled,y_train)
+ y_pred = model.predict(X_test_scaled)
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
+ variance[degree] = np.mean( np.var(y_pred) )
+ print('Max depth:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
+
+
Xgboost on the Cancer Data
diff --git a/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb b/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
index 59ba61dcb..c27d3d1be 100644
--- a/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
+++ b/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
@@ -1796,6 +1796,74 @@
"plt.show()"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Changing the Level of the Decision Tree"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "\n",
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "from sklearn.pipeline import make_pipeline\n",
+ "from sklearn.utils import resample\n",
+ "from sklearn.tree import DecisionTreeRegressor\n",
+ "\n",
+ "n = 100\n",
+ "n_boostraps = 100\n",
+ "maxdepth = 8\n",
+ "\n",
+ "# Make data set.\n",
+ "x = np.linspace(-3, 3, n).reshape(-1, 1)\n",
+ "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n",
+ "error = np.zeros(maxdepth)\n",
+ "bias = np.zeros(maxdepth)\n",
+ "variance = np.zeros(maxdepth)\n",
+ "polydegree = np.zeros(maxdepth)\n",
+ "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
+ "\n",
+ "from sklearn.preprocessing import StandardScaler\n",
+ "scaler = StandardScaler()\n",
+ "scaler.fit(X_train)\n",
+ "X_train_scaled = scaler.transform(X_train)\n",
+ "X_test_scaled = scaler.transform(X_test)\n",
+ "\n",
+ "for degree in range(1,maxdepth):\n",
+ " model = DecisionTreeRegressor(max_depth=degree) \n",
+ " y_pred = np.empty((y_test.shape[0], n_boostraps))\n",
+ " for i in range(n_boostraps):\n",
+ " x_, y_ = resample(X_train_scaled, y_train)\n",
+ " model.fit(x_, y_)\n",
+ " y_pred[:, i] = model.predict(X_test_scaled)#.ravel()\n",
+ "\n",
+ " polydegree[degree] = degree\n",
+ " error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )\n",
+ " bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )\n",
+ " variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )\n",
+ " print('Polynomial degree:', degree)\n",
+ " print('Error:', error[degree])\n",
+ " print('Bias^2:', bias[degree])\n",
+ " print('Var:', variance[degree])\n",
+ " print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))\n",
+ "\n",
+ "plt.xlim(1,maxdepth)\n",
+ "plt.plot(polydegree, error, label='Error')\n",
+ "plt.plot(polydegree, bias, label='bias')\n",
+ "plt.plot(polydegree, variance, label='Variance')\n",
+ "plt.legend()\n",
+ "plt.show()"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -1872,7 +1940,7 @@
},
{
"cell_type": "code",
- "execution_count": 25,
+ "execution_count": 26,
"metadata": {
"collapsed": false
},
@@ -1954,7 +2022,7 @@
},
{
"cell_type": "code",
- "execution_count": 26,
+ "execution_count": 27,
"metadata": {
"collapsed": false
},
@@ -1967,7 +2035,7 @@
},
{
"cell_type": "code",
- "execution_count": 27,
+ "execution_count": 28,
"metadata": {
"collapsed": false
},
@@ -1986,12 +2054,12 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "## Bootstrap with Random Forests Instead of a Single Tree"
+ "## Bootstrap with Random Forests Instead of a Single Tree, own Bagging"
]
},
{
"cell_type": "code",
- "execution_count": 28,
+ "execution_count": 29,
"metadata": {
"collapsed": false
},
@@ -2007,7 +2075,7 @@
"\n",
"np.random.seed(2018)\n",
"\n",
- "n = 40\n",
+ "n = 100\n",
"n_boostraps = 100\n",
"maxdegree = 14\n",
"\n",
@@ -2031,7 +2099,7 @@
" y_pred = np.empty((y_test.shape[0], n_boostraps))\n",
" for i in range(n_boostraps):\n",
" x_, y_ = resample(X_train_scaled, y_train)\n",
- " model.fit(x_, y_)\n",
+ " model.fit(x_, y_.ravel())\n",
" y_pred[:, i] = model.predict(X_test_scaled).ravel()\n",
"\n",
" polydegree[degree] = degree\n",
@@ -2120,7 +2188,7 @@
"source": [
"1. Then we start looping over all attempts at classifying, namely we start an iterative process for $m=1:M$, where $M$ is the final number of classifications. Our given classifier could for example be a plain decision tree.\n",
"\n",
- "a. Fit thus a given classifier to the training using the weights $w_i$.\n",
+ "a. Fit then a given classifier to the training using the weights $w_i$.\n",
"\n",
"b. Compute then $\\mathrm{err}$ and figure out which events are classified properly and which are classified wrongly.\n",
"\n",
@@ -2139,7 +2207,6 @@
"new classification step $m$ is then forced to concentrate on those\n",
"observations that are missed in the previous iterations.\n",
"\n",
- "## Figure to Illustrate the Iterative Classification Process\n",
"\n",
"\n",
"## AdaBoost Examples\n",
@@ -2149,7 +2216,7 @@
},
{
"cell_type": "code",
- "execution_count": 29,
+ "execution_count": 30,
"metadata": {
"collapsed": false
},
@@ -2239,7 +2306,7 @@
},
{
"cell_type": "code",
- "execution_count": 30,
+ "execution_count": 31,
"metadata": {
"collapsed": false
},
@@ -2338,7 +2405,7 @@
},
{
"cell_type": "code",
- "execution_count": 31,
+ "execution_count": 32,
"metadata": {
"collapsed": false
},
@@ -2423,12 +2490,75 @@
"\n",
"It is now the algorithm which wins essentially all ML competitions!!!\n",
"\n",
+ "## Regression Case"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "import xgboost as xgb\n",
+ "from sklearn.preprocessing import StandardScaler\n",
+ "import scikitplot as skplt\n",
+ "from sklearn.metrics import mean_squared_error\n",
+ "\n",
+ "n = 40\n",
+ "n_boostraps = 100\n",
+ "maxdegree = 8\n",
+ "\n",
+ "# Make data set.\n",
+ "x = np.linspace(-3, 3, n).reshape(-1, 1)\n",
+ "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n",
+ "\n",
+ "error = np.zeros(maxdegree)\n",
+ "bias = np.zeros(maxdegree)\n",
+ "variance = np.zeros(maxdegree)\n",
+ "polydegree = np.zeros(maxdegree)\n",
+ "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
+ "scaler = StandardScaler()\n",
+ "scaler.fit(X_train)\n",
+ "X_train_scaled = scaler.transform(X_train)\n",
+ "X_test_scaled = scaler.transform(X_test)\n",
+ "\n",
+ "for degree in range(maxdegree):\n",
+ " model = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1,\n",
+ " max_depth = maxdegree, alpha = 10, n_estimators = 10)\n",
+ " model.fit(X_train_scaled,y_train)\n",
+ " y_pred = model.predict(X_test_scaled)\n",
+ " polydegree[degree] = degree\n",
+ " error[degree] = np.mean( np.mean((y_test - y_pred)**2) )\n",
+ " bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )\n",
+ " variance[degree] = np.mean( np.var(y_pred) )\n",
+ " print('Max depth:', degree)\n",
+ " print('Error:', error[degree])\n",
+ " print('Bias^2:', bias[degree])\n",
+ " print('Var:', variance[degree])\n",
+ " print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))\n",
+ "\n",
+ "plt.plot(polydegree, error, label='Error')\n",
+ "plt.plot(polydegree, bias, label='bias')\n",
+ "plt.plot(polydegree, variance, label='Variance')\n",
+ "plt.legend()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
"## Xgboost on the Cancer Data"
]
},
{
"cell_type": "code",
- "execution_count": 32,
+ "execution_count": 34,
"metadata": {
"collapsed": false
},
diff --git a/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz b/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz
index 10e484876..69d0000cf 100644
Binary files a/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz and b/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz differ
diff --git a/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf b/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf
index c5c80b215..9139b87ab 100644
Binary files a/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf and b/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf differ
diff --git a/doc/src/DecisionTrees/DecisionTrees.do.txt b/doc/src/DecisionTrees/DecisionTrees.do.txt
index 8926c0e71..ac948a01c 100644
--- a/doc/src/DecisionTrees/DecisionTrees.do.txt
+++ b/doc/src/DecisionTrees/DecisionTrees.do.txt
@@ -1449,6 +1449,69 @@ plt.show()
!ec
+!split
+===== Changing the Level of the Decision Tree =====
+
+!bc pycod
+
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
+
+n = 100
+n_boostraps = 100
+maxdepth = 8
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdepth)
+bias = np.zeros(maxdepth)
+variance = np.zeros(maxdepth)
+polydegree = np.zeros(maxdepth)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(1,maxdepth):
+ model = DecisionTreeRegressor(max_depth=degree)
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled)#.ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.xlim(1,maxdepth)
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
+
+!ec
+
+
+
!split
===== Random forests =====
@@ -1599,7 +1662,7 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
!split
-===== Bootstrap with Random Forests Instead of a Single Tree =====
+===== Bootstrap with Random Forests Instead of a Single Tree, own Bagging =====
!bc pycod
@@ -1612,7 +1675,7 @@ from sklearn.ensemble import RandomForestRegressor
np.random.seed(2018)
-n = 40
+n = 100
n_boostraps = 100
maxdegree = 14
@@ -1636,7 +1699,7 @@ for degree in range(maxdegree):
y_pred = np.empty((y_test.shape[0], n_boostraps))
for i in range(n_boostraps):
x_, y_ = resample(X_train_scaled, y_train)
- model.fit(x_, y_)
+ model.fit(x_, y_.ravel())
y_pred[:, i] = model.predict(X_test_scaled).ravel()
polydegree[degree] = degree
@@ -1707,7 +1770,7 @@ o We rewrite the misclassification error as
\]
!et
o Then we start looping over all attempts at classifying, namely we start an iterative process for $m=1:M$, where $M$ is the final number of classifications. Our given classifier could for example be a plain decision tree.
- o Fit thus a given classifier to the training using the weights $w_i$.
+ o Fit then a given classifier to the training using the weights $w_i$.
o Compute then $\mathrm{err}$ and figure out which events are classified properly and which are classified wrongly.
o Define a quantity $\alpha_{m} = \log{(1-\mathrm{err})/\mathrm{err}}
o Set the new weights to $w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(\bm{X}_{i*})}.
@@ -1721,8 +1784,6 @@ difficult to classifiy correctly are given a larger influence. Each
new classification step $m$ is then forced to concentrate on those
observations that are missed in the previous iterations.
-!split
-===== Figure to Illustrate the Iterative Classification Process =====
!split
@@ -1965,6 +2026,62 @@ sketch for efficient proposal calculation. It introduces a novel sparsity-aware
It is now the algorithm which wins essentially all ML competitions!!!
+!split
+===== Regression Case =====
+
+!bc pycod
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+import xgboost as xgb
+from sklearn.preprocessing import StandardScaler
+import scikitplot as skplt
+from sklearn.metrics import mean_squared_error
+
+n = 40
+n_boostraps = 100
+maxdegree = 8
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1,
+ max_depth = maxdegree, alpha = 10, n_estimators = 10)
+ model.fit(X_train_scaled,y_train)
+ y_pred = model.predict(X_test_scaled)
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
+ variance[degree] = np.mean( np.var(y_pred) )
+ print('Max depth:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+!ec
+
+
+
!split
===== Xgboost on the Cancer Data =====
!bc pycod
diff --git a/doc/src/DecisionTrees/Programs/bootstrap.py~ b/doc/src/DecisionTrees/Programs/bootstrap.py~
new file mode 100644
index 000000000..623de3af2
--- /dev/null
+++ b/doc/src/DecisionTrees/Programs/bootstrap.py~
@@ -0,0 +1,56 @@
+
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
+
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = DecisionTreeRegressor(max_depth=2)
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
diff --git a/doc/src/DecisionTrees/Programs/rfbootclassify.py b/doc/src/DecisionTrees/Programs/rfbootclassify.py
new file mode 100644
index 000000000..6bd25a532
--- /dev/null
+++ b/doc/src/DecisionTrees/Programs/rfbootclassify.py
@@ -0,0 +1,54 @@
+
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
+
+n = 100
+n_boostraps = 100
+maxdepth = 8
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdepth)
+bias = np.zeros(maxdepth)
+variance = np.zeros(maxdepth)
+polydegree = np.zeros(maxdepth)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(1,maxdepth):
+ model = DecisionTreeRegressor(max_depth=degree)
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled)#.ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.xlim(1,maxdepth)
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
diff --git a/doc/src/DecisionTrees/Programs/rfbootclassify.py~ b/doc/src/DecisionTrees/Programs/rfbootclassify.py~
new file mode 100644
index 000000000..84994c400
--- /dev/null
+++ b/doc/src/DecisionTrees/Programs/rfbootclassify.py~
@@ -0,0 +1,54 @@
+
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.tree import DecisionTreeRegressor
+
+n = 100
+n_boostraps = 100
+maxdepth = 8
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdepth)
+bias = np.zeros(maxdepth)
+variance = np.zeros(maxdepth)
+polydegree = np.zeros(maxdepth)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(1,maxdepth):
+ model = DecisionTreeRegressor(max_depth=degree)
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train)
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled)#.ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.xlim(1,maxdegree)
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
+
diff --git a/doc/src/DecisionTrees/Programs/rfbootstrap.py b/doc/src/DecisionTrees/Programs/rfbootstrap.py
index 824862286..db31345b5 100644
--- a/doc/src/DecisionTrees/Programs/rfbootstrap.py
+++ b/doc/src/DecisionTrees/Programs/rfbootstrap.py
@@ -8,13 +8,14 @@ from sklearn.ensemble import RandomForestRegressor
np.random.seed(2018)
-n = 40
+n = 500
n_boostraps = 100
maxdegree = 14
# Make data set.
x = np.linspace(-3, 3, n).reshape(-1, 1)
y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+
error = np.zeros(maxdegree)
bias = np.zeros(maxdegree)
variance = np.zeros(maxdegree)
@@ -32,8 +33,8 @@ for degree in range(maxdegree):
y_pred = np.empty((y_test.shape[0], n_boostraps))
for i in range(n_boostraps):
x_, y_ = resample(X_train_scaled, y_train)
- model.fit(x_, y_)
- y_pred[:, i] = model.predict(X_test_scaled)#.ravel()
+ model.fit(x_, y_.ravel())
+ y_pred[:, i] = model.predict(X_test_scaled)
polydegree[degree] = degree
error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
diff --git a/doc/src/DecisionTrees/Programs/rfbootstrap.py~ b/doc/src/DecisionTrees/Programs/rfbootstrap.py~
new file mode 100644
index 000000000..96f81e3df
--- /dev/null
+++ b/doc/src/DecisionTrees/Programs/rfbootstrap.py~
@@ -0,0 +1,54 @@
+
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+from sklearn.ensemble import RandomForestRegressor
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = RandomForestRegressor()
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(X_train_scaled, y_train).ravel()
+ model.fit(x_, y_)
+ y_pred[:, i] = model.predict(X_test_scaled).ravel()
+
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+
diff --git a/doc/src/DecisionTrees/Programs/rfcancer.py b/doc/src/DecisionTrees/Programs/rfcancer.py
index 9f226f064..d507268cb 100644
--- a/doc/src/DecisionTrees/Programs/rfcancer.py
+++ b/doc/src/DecisionTrees/Programs/rfcancer.py
@@ -29,8 +29,6 @@ accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_
print(accuracy)
print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test)))
-
-
y_pred = Random_Forest_model.predict(X_test_scaled)
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
plt.show()
diff --git a/doc/src/DecisionTrees/Programs/xgcancer.py~ b/doc/src/DecisionTrees/Programs/xgcancer.py~
new file mode 100644
index 000000000..9f226f064
--- /dev/null
+++ b/doc/src/DecisionTrees/Programs/xgcancer.py~
@@ -0,0 +1,41 @@
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+import scikitplot as skplt
+
+# Load the data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+# Data set not specificied
+#Instantiate the model with 100 trees and entropy as splitting criteria
+Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy")
+Random_Forest_model.fit(X_train_scaled, y_train)
+#Cross validation
+accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score']
+print(accuracy)
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test)))
+
+
+
+y_pred = Random_Forest_model.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = Random_Forest_model.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
diff --git a/doc/src/DecisionTrees/Programs/xgregressor.py b/doc/src/DecisionTrees/Programs/xgregressor.py
new file mode 100644
index 000000000..0aaf7be2f
--- /dev/null
+++ b/doc/src/DecisionTrees/Programs/xgregressor.py
@@ -0,0 +1,47 @@
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+import xgboost as xgb
+from sklearn.preprocessing import StandardScaler
+import scikitplot as skplt
+from sklearn.metrics import mean_squared_error
+
+n = 40
+n_boostraps = 100
+maxdegree = 8
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1,
+ max_depth = maxdegree, alpha = 10, n_estimators = 10)
+ model.fit(X_train_scaled,y_train)
+ y_pred = model.predict(X_test_scaled)
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
+ variance[degree] = np.mean( np.var(y_pred) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
diff --git a/doc/src/DecisionTrees/Programs/xgregressor.py~ b/doc/src/DecisionTrees/Programs/xgregressor.py~
new file mode 100644
index 000000000..6c9b2a16d
--- /dev/null
+++ b/doc/src/DecisionTrees/Programs/xgregressor.py~
@@ -0,0 +1,47 @@
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+import xgboost as xgb
+from sklearn.preprocessing import StandardScaler
+import scikitplot as skplt
+from sklearn.metrics import mean_squared_error
+
+n = 500
+n_boostraps = 100
+maxdegree = 8
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+for degree in range(maxdegree):
+ model = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1,
+ max_depth = maxdegree, alpha = 10, n_estimators = 10)
+ model.fit(X_train_scaled,y_train)
+ y_pred = model.predict(X_test_scaled)
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
+ variance[degree] = np.mean( np.var(y_pred) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+