added xgboost example

doc/src/DecisionTrees/DecisionTrees.do.txt

@@ -1164,6 +1164,14 @@ o Boosting methods
 
 We discuss these methods here.
 
+
+!split
+===== An Overview of Ensemble Methods  =====
+
+FIGURE: [DataFiles/ensembleoverview.png, width=600 frac=0.8]
+
+
+
 !split
 ===== Bagging =====
 
@@ -1832,4 +1840,10 @@ Boosting, is an optimized distributed gradient boosting library
 designed to be highly efficient, flexible and portable. It implements
 machine learning algorithms under the Gradient Boosting
 framework. XGBoost provides a parallel tree boosting that solve many
-data science problems in a fast and accurate way
+data science problems in a fast and accurate way. See the "article by Chen and Guestrin":"https://arxiv.org/abs/1603.02754".
+
+The authors design and build a highly scalable end-to-end tree
+boosting system. It has  a theoretically justified weighted quantile
+sketch for efficient proposal calculation. It introduces a novel sparsity-aware algorithm for parallel tree learning and an effective cache-aware block structure for out-of-core tree learning.
+
+It is now the algorithm which wins essentially all ML competitions!!!

doc/src/DecisionTrees/Programs/rfcancer.py

@@ -2,9 +2,11 @@ import matplotlib.pyplot as plt
 import numpy as np
 from sklearn.model_selection import  train_test_split 
 from sklearn.datasets import load_breast_cancer
-from sklearn.svm import SVC
-from sklearn.linear_model import LogisticRegression
 from sklearn.tree import DecisionTreeClassifier
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+import scikitplot as skplt
 
 # Load the data
 cancer = load_breast_cancer()
@@ -12,38 +14,12 @@ cancer = load_breast_cancer()
 X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
 print(X_train.shape)
 print(X_test.shape)
-# Logistic Regression
-logreg = LogisticRegression(solver='lbfgs')
-logreg.fit(X_train, y_train)
-print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
-# Support vector machine
-svm = SVC(gamma='auto', C=100)
-svm.fit(X_train, y_train)
-print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
-# Decision Trees
-deep_tree_clf = DecisionTreeClassifier(max_depth=None)
-deep_tree_clf.fit(X_train, y_train)
-print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
 #now scale the data
 from sklearn.preprocessing import StandardScaler
 scaler = StandardScaler()
 scaler.fit(X_train)
 X_train_scaled = scaler.transform(X_train)
 X_test_scaled = scaler.transform(X_test)
-# Logistic Regression
-logreg.fit(X_train_scaled, y_train)
-print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Support Vector Machine
-svm.fit(X_train_scaled, y_train)
-print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Decision Trees
-deep_tree_clf.fit(X_train_scaled, y_train)
-print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
-
-
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.preprocessing import LabelEncoder
-from sklearn.model_selection import cross_validate
 # Data set not specificied
 #Instantiate the model with 100 trees and entropy as splitting criteria
 Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy")
@@ -54,10 +30,9 @@ print(accuracy)
 print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test)))
 
 
-import scikitplot as skplt
+
 y_pred = Random_Forest_model.predict(X_test_scaled)
 skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
-#<matplotlib.axes._subplots.AxesSubplot object at 0x7fe967d64490>
 plt.show()
 y_probas = Random_Forest_model.predict_proba(X_test_scaled)
 skplt.metrics.plot_roc(y_test, y_probas)