-
@@ -234,7 +257,7 @@ MathJax.Hub.Config({
+Figure to come here. +
@@ -211,7 +237,7 @@ MathJax.Hub.Config({
-The example we will look at is a classical one in many Machine -Learning applications. Based on various meteorological features, we -have several so-called attributes which decide whether we at the end -will do some outdoor activity like skiing, going for a bike ride etc -etc. The table here contains the feautures outlook, temperature, -humidity and wind. The target or output is whether we ride -(True=1) or whether we do something else that day (False=0). The -attributes for each feature are then sunny, overcast and rain for the -outlook, hot, cold and mild for temperature, high and normal for -humidity and weak and strong for wind. +
-The table here summarizes the various attributes and +We discuss both algorithms with applications here. The popular library -Scikit-Learn_ uses the CART algorithm. For classification problems you can use either the gini index or the entropy to split a tree in two branches. -
| Day | Outlook | Temperature | Humidity | Wind | Ride |
| 1 | Sunny | Hot | High | Weak | 0 |
| 2 | Sunny | Hot | High | Strong | 1 |
| 3 | Overcast | Hot | High | Weak | 1 |
| 4 | Rain | Mild | High | Weak | 1 |
| 5 | Rain | Cool | Normal | Weak | 1 |
| 6 | Rain | Cool | Normal | Strong | 0 |
| 7 | Overcast | Cool | Normal | Strong | 1 |
| 8 | Sunny | Mild | High | Weak | 0 |
| 9 | Sunny | Cool | Normal | Weak | 1 |
| 10 | Rain | Mild | Normal | Weak | 1 |
| 11 | Sunny | Mild | Normal | Strong | 1 |
| 12 | Overcast | Mild | High | Strong | 1 |
| 13 | Overcast | Hot | Normal | Weak | 1 |
| 14 | Rain | Mild | High | Strong | 0 |
@@ -259,7 +250,7 @@ The table here summarizes the various attributes and
- - -
# Common imports
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.tree import export_graphviz
-from sklearn.preprocessing import StandardScaler, OneHotEncoder
-from sklearn.compose import ColumnTransformer
-from IPython.display import Image
-from pydot import graph_from_dot_data
-import os
-
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
-
-if not os.path.exists(PROJECT_ROOT_DIR):
- os.mkdir(PROJECT_ROOT_DIR)
-
-if not os.path.exists(FIGURE_ID):
- os.makedirs(FIGURE_ID)
-
-if not os.path.exists(DATA_ID):
- os.makedirs(DATA_ID)
-
-def image_path(fig_id):
- return os.path.join(FIGURE_ID, fig_id)
-
-def data_path(dat_id):
- return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
- plt.savefig(image_path(fig_id) + ".png", format='png')
-
-infile = open(data_path("rideclass.csv"),'r')
-
-# Read the experimental data with Pandas
-from IPython.display import display
-ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
-ridedata = pd.DataFrame(ridedata)
-
-# Features and targets
-X = ridedata.loc[:, ridedata.columns != 'Ride'].values
-y = ridedata.loc[:, ridedata.columns == 'Ride'].values
-
-# Create the encoder.
-encoder = OneHotEncoder(handle_unknown="ignore")
-# Assume for simplicity all features are categorical.
-encoder.fit(X)
-# Apply the encoder.
-X = encoder.transform(X)
-print(X)
-# Then do a Classification tree
-tree_clf = DecisionTreeClassifier(max_depth=2)
-tree_clf.fit(X, y)
-print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
-#transfer to a decision tree graph
-export_graphviz(
- tree_clf,
- out_file="DataFiles/ride.dot",
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
-os.system(cmd)
-
@@ -290,7 +242,7 @@ os.system(cmd)
-The above functions (gini, entropy and misclassification error) are -important components of the so-called CART algorithm. We will discuss -this algorithm below after we have discussed the information gain -algorithm ID3. - -
-In the example here we have converted all our attributes into numerical values \( 0,1,2 \) etc. - -
- - -
# Split a dataset based on an attribute and an attribute value
-def test_split(index, value, dataset):
- left, right = list(), list()
- for row in dataset:
- if row[index] < value:
- left.append(row)
- else:
- right.append(row)
- return left, right
-
-# Calculate the Gini index for a split dataset
-def gini_index(groups, classes):
- # count all samples at split point
- n_instances = float(sum([len(group) for group in groups]))
- # sum weighted Gini index for each group
- gini = 0.0
- for group in groups:
- size = float(len(group))
- # avoid divide by zero
- if size == 0:
- continue
- score = 0.0
- # score the group based on the score for each class
- for class_val in classes:
- p = [row[-1] for row in group].count(class_val) / size
- score += p * p
- # weight the group score by its relative size
- gini += (1.0 - score) * (size / n_instances)
- return gini
-
-# Select the best split point for a dataset
-def get_split(dataset):
- class_values = list(set(row[-1] for row in dataset))
- b_index, b_value, b_score, b_groups = 999, 999, 999, None
- for index in range(len(dataset[0])-1):
- for row in dataset:
- groups = test_split(index, row[index], dataset)
- gini = gini_index(groups, class_values)
- print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini))
- if gini < b_score:
- b_index, b_value, b_score, b_groups = index, row[index], gini, groups
- return {'index':b_index, 'value':b_value, 'groups':b_groups}
-
-dataset = [[0,0,0,0,0],
- [0,0,0,1,1],
- [1,0,0,0,1],
- [2,1,0,0,1],
- [2,2,1,0,1],
- [2,2,1,1,0],
- [1,2,1,1,1],
- [0,1,0,0,0],
- [0,2,1,0,1],
- [2,1,1,0,1],
- [0,1,1,1,1],
- [1,1,0,1,1],
- [1,0,1,0,1],
- [2,1,0,1,0]]
-
-split = get_split(dataset)
-print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
-
@@ -292,7 +242,7 @@ split = get_split(dataset)
-ID3, learns decision trees by constructing -them topdown, beginning with the question which attribute should be tested at the root of the tree? - -
-We would like to select the attribute that is most useful for classifying -examples. - -
-What is a good quantitative measure of the worth of an attribute? - -
-Information gain measures how well a given attribute separates the -training examples according to their target classification. - -
-The ID3 algorithm uses this information gain measure to select among the candidate -attributes at each step while growing the tree. +The table here summarizes the various attributes and +
| Day | Outlook | Temperature | Humidity | Wind | Ride |
| 1 | Sunny | Hot | High | Weak | 0 |
| 2 | Sunny | Hot | High | Strong | 1 |
| 3 | Overcast | Hot | High | Weak | 1 |
| 4 | Rain | Mild | High | Weak | 1 |
| 5 | Rain | Cool | Normal | Weak | 1 |
| 6 | Rain | Cool | Normal | Strong | 0 |
| 7 | Overcast | Cool | Normal | Strong | 1 |
| 8 | Sunny | Mild | High | Weak | 0 |
| 9 | Sunny | Cool | Normal | Weak | 1 |
| 10 | Rain | Mild | Normal | Weak | 1 |
| 11 | Sunny | Mild | Normal | Strong | 1 |
| 12 | Overcast | Mild | High | Strong | 1 |
| 13 | Overcast | Hot | Normal | Weak | 1 |
| 14 | Rain | Mild | High | Strong | 0 |
@@ -250,7 +282,7 @@ attributes at each step while growing the tree.
-
import re
-import math
-from collections import deque
+# Common imports
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.tree import export_graphviz
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from IPython.display import Image
+from pydot import graph_from_dot_data
+import os
-# x is examples in training set
-# y is set of targets
-# label is target attributes
-# Node is a class which has properties values, childs, and next
-# root is top node in the decision tree
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
-class Node(object):
- def __init__(self):
- self.value = None
- self.next = None
- self.childs = None
+if not os.path.exists(PROJECT_ROOT_DIR):
+ os.mkdir(PROJECT_ROOT_DIR)
-# Simple class of Decision Tree
-# Aimed for who want to learn Decision Tree, so it is not optimized
-class DecisionTree(object):
- def __init__(self, sample, attributes, labels):
- self.sample = sample
- self.attributes = attributes
- self.labels = labels
- self.labelCodes = None
- self.labelCodesCount = None
- self.initLabelCodes()
- # print(self.labelCodes)
- self.root = None
- self.entropy = self.getEntropy([x for x in range(len(self.labels))])
+if not os.path.exists(FIGURE_ID):
+ os.makedirs(FIGURE_ID)
- def initLabelCodes(self):
- self.labelCodes = []
- self.labelCodesCount = []
- for l in self.labels:
- if l not in self.labelCodes:
- self.labelCodes.append(l)
- self.labelCodesCount.append(0)
- self.labelCodesCount[self.labelCodes.index(l)] += 1
+if not os.path.exists(DATA_ID):
+ os.makedirs(DATA_ID)
- def getLabelCodeId(self, sampleId):
- return self.labelCodes.index(self.labels[sampleId])
+def image_path(fig_id):
+ return os.path.join(FIGURE_ID, fig_id)
- def getAttributeValues(self, sampleIds, attributeId):
- vals = []
- for sid in sampleIds:
- val = self.sample[sid][attributeId]
- if val not in vals:
- vals.append(val)
- # print(vals)
- return vals
+def data_path(dat_id):
+ return os.path.join(DATA_ID, dat_id)
- def getEntropy(self, sampleIds):
- entropy = 0
- labelCount = [0] * len(self.labelCodes)
- for sid in sampleIds:
- labelCount[self.getLabelCodeId(sid)] += 1
- # print("-ge", labelCount)
- for lv in labelCount:
- # print(lv)
- if lv != 0:
- entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
- else:
- entropy += 0
- return entropy
+def save_fig(fig_id):
+ plt.savefig(image_path(fig_id) + ".png", format='png')
- def getDominantLabel(self, sampleIds):
- labelCodesCount = [0] * len(self.labelCodes)
- for sid in sampleIds:
- labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
- return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
+infile = open(data_path("rideclass.csv"),'r')
- def getInformationGain(self, sampleIds, attributeId):
- gain = self.getEntropy(sampleIds)
- attributeVals = []
- attributeValsCount = []
- attributeValsIds = []
- for sid in sampleIds:
- val = self.sample[sid][attributeId]
- if val not in attributeVals:
- attributeVals.append(val)
- attributeValsCount.append(0)
- attributeValsIds.append([])
- vid = attributeVals.index(val)
- attributeValsCount[vid] += 1
- attributeValsIds[vid].append(sid)
- # print("-gig", self.attributes[attributeId])
- for vc, vids in zip(attributeValsCount, attributeValsIds):
- # print("-gig", vids)
- gain -= vc/len(sampleIds) * self.getEntropy(vids)
- return gain
+# Read the experimental data with Pandas
+from IPython.display import display
+ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
+ridedata = pd.DataFrame(ridedata)
- def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
- attributesEntropy = [0] * len(attributeIds)
- for i, attId in zip(range(len(attributeIds)), attributeIds):
- attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
- maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
- return self.attributes[maxId], maxId
+# Features and targets
+X = ridedata.loc[:, ridedata.columns != 'Ride'].values
+y = ridedata.loc[:, ridedata.columns == 'Ride'].values
- def isSingleLabeled(self, sampleIds):
- label = self.labels[sampleIds[0]]
- for sid in sampleIds:
- if self.labels[sid] != label:
- return False
- return True
-
- def getLabel(self, sampleId):
- return self.labels[sampleId]
-
- def id3(self):
- sampleIds = [x for x in range(len(self.sample))]
- attributeIds = [x for x in range(len(self.attributes))]
- self.root = self.id3Recv(sampleIds, attributeIds, self.root)
-
- def id3Recv(self, sampleIds, attributeIds, root):
- root = Node() # Initialize current root
- if self.isSingleLabeled(sampleIds):
- root.value = self.labels[sampleIds[0]]
- return root
- # print(attributeIds)
- if len(attributeIds) == 0:
- root.value = self.getDominantLabel(sampleIds)
- return root
- bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
- sampleIds, attributeIds)
- # print(bestAttrName)
- root.value = bestAttrName
- root.childs = [] # Create list of children
- for value in self.getAttributeValues(sampleIds, bestAttrId):
- # print(value)
- child = Node()
- child.value = value
- root.childs.append(child) # Append new child node to current
- # root
- childSampleIds = []
- for sid in sampleIds:
- if self.sample[sid][bestAttrId] == value:
- childSampleIds.append(sid)
- if len(childSampleIds) == 0:
- child.next = self.getDominantLabel(sampleIds)
- else:
- # print(bestAttrName, bestAttrId)
- # print(attributeIds)
- if len(attributeIds) > 0 and bestAttrId in attributeIds:
- toRemove = attributeIds.index(bestAttrId)
- attributeIds.pop(toRemove)
- child.next = self.id3Recv(
- childSampleIds, attributeIds, child.next)
- return root
-
- def printTree(self):
- if self.root:
- roots = deque()
- roots.append(self.root)
- while len(roots) > 0:
- root = roots.popleft()
- print(root.value)
- if root.childs:
- for child in root.childs:
- print('({})'.format(child.value))
- roots.append(child.next)
- elif root.next:
- print(root.next)
-
-
-def test():
- f = open('DataFiles/rideclass.csv')
- attributes = f.readline().split(',')
- attributes = attributes[1:len(attributes)-1]
- print(attributes)
- sample = f.readlines()
- f.close()
- for i in range(len(sample)):
- sample[i] = re.sub('\d+,', '', sample[i])
- sample[i] = sample[i].strip().split(',')
- labels = []
- for s in sample:
- labels.append(s.pop())
- # print(sample)
- # print(labels)
- decisionTree = DecisionTree(sample, attributes, labels)
- print("System entropy {}".format(decisionTree.entropy))
- decisionTree.id3()
- decisionTree.printTree()
-
-
-if __name__ == '__main__':
- test()
+# Create the encoder.
+encoder = OneHotEncoder(handle_unknown="ignore")
+# Assume for simplicity all features are categorical.
+encoder.fit(X)
+# Apply the encoder.
+X = encoder.transform(X)
+print(X)
+# Then do a Classification tree
+tree_clf = DecisionTreeClassifier(max_depth=2)
+tree_clf.fit(X, y)
+print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
+#transfer to a decision tree graph
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/ride.dot",
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
+os.system(cmd)
@@ -410,7 +313,7 @@ MathJax.Hub.Config({
+The above functions (gini, entropy and misclassification error) are +important components of the so-called CART algorithm. We will discuss +this algorithm below after we have discussed the information gain +algorithm ID3. + +
+In the example here we have converted all our attributes into numerical values \( 0,1,2 \) etc. +
-
import matplotlib.pyplot as plt
-import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn.datasets import load_breast_cancer
-from sklearn.svm import SVC
-from sklearn.linear_model import LogisticRegression
-from sklearn.tree import DecisionTreeClassifier
+# Split a dataset based on an attribute and an attribute value
+def test_split(index, value, dataset):
+ left, right = list(), list()
+ for row in dataset:
+ if row[index] < value:
+ left.append(row)
+ else:
+ right.append(row)
+ return left, right
+
+# Calculate the Gini index for a split dataset
+def gini_index(groups, classes):
+ # count all samples at split point
+ n_instances = float(sum([len(group) for group in groups]))
+ # sum weighted Gini index for each group
+ gini = 0.0
+ for group in groups:
+ size = float(len(group))
+ # avoid divide by zero
+ if size == 0:
+ continue
+ score = 0.0
+ # score the group based on the score for each class
+ for class_val in classes:
+ p = [row[-1] for row in group].count(class_val) / size
+ score += p * p
+ # weight the group score by its relative size
+ gini += (1.0 - score) * (size / n_instances)
+ return gini
-# Load the data
-cancer = load_breast_cancer()
+# Select the best split point for a dataset
+def get_split(dataset):
+ class_values = list(set(row[-1] for row in dataset))
+ b_index, b_value, b_score, b_groups = 999, 999, 999, None
+ for index in range(len(dataset[0])-1):
+ for row in dataset:
+ groups = test_split(index, row[index], dataset)
+ gini = gini_index(groups, class_values)
+ print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini))
+ if gini < b_score:
+ b_index, b_value, b_score, b_groups = index, row[index], gini, groups
+ return {'index':b_index, 'value':b_value, 'groups':b_groups}
+
+dataset = [[0,0,0,0,0],
+ [0,0,0,1,1],
+ [1,0,0,0,1],
+ [2,1,0,0,1],
+ [2,2,1,0,1],
+ [2,2,1,1,0],
+ [1,2,1,1,1],
+ [0,1,0,0,0],
+ [0,2,1,0,1],
+ [2,1,1,0,1],
+ [0,1,1,1,1],
+ [1,1,0,1,1],
+ [1,0,1,0,1],
+ [2,1,0,1,0]]
-X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-print(X_train.shape)
-print(X_test.shape)
-# Logistic Regression
-logreg = LogisticRegression(solver='lbfgs')
-logreg.fit(X_train, y_train)
-print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
-# Support vector machine
-svm = SVC(gamma='auto', C=100)
-svm.fit(X_train, y_train)
-print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
-# Decision Trees
-deep_tree_clf = DecisionTreeClassifier(max_depth=None)
-deep_tree_clf.fit(X_train, y_train)
-print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
-#now scale the data
-from sklearn.preprocessing import StandardScaler
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-# Logistic Regression
-logreg.fit(X_train_scaled, y_train)
-print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Support Vector Machine
-svm.fit(X_train_scaled, y_train)
-print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Decision Trees
-deep_tree_clf.fit(X_train_scaled, y_train)
-print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
+split = get_split(dataset)
+print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
@@ -263,7 +315,7 @@ deep_tree_clf.fit(X_train_scaled, y_train)
+ID3, learns decision trees by constructing +them topdown, beginning with the question which attribute should be tested at the root of the tree? - -
from __future__ import division, print_function, unicode_literals
+
+- Each instance attribute is evaluated using a statistical test to determine how well it alone classifies the training examples.
+- The best attribute is selected and used as the test at the root node of the tree.
+- A descendant of the root node is then created for each possible value of this attribute.
+- Training examples are sorted to the appropriate descendant node.
+- The entire process is then repeated using the training examples associated with each descendant node to select the best attribute to test at that point in the tree.
+- This forms a greedy search for an acceptable decision tree, in which the algorithm never backtracks to reconsider earlier choices.
+
-# Common imports
-import numpy as np
-import os
+The ID3 algorithm selects, which attribute to test at each node in the
+tree.
-# to make this notebook's output stable across runs
-np.random.seed(42)
+
+We would like to select the attribute that is most useful for classifying
+examples.
-# To plot pretty figures
-import matplotlib
-import matplotlib.pyplot as plt
-from matplotlib.colors import ListedColormap
-plt.rcParams['axes.labelsize'] = 14
-plt.rcParams['xtick.labelsize'] = 12
-plt.rcParams['ytick.labelsize'] = 12
+
+What is a good quantitative measure of the worth of an attribute?
+
+Information gain measures how well a given attribute separates the
+training examples according to their target classification.
-from sklearn.svm import SVC
-from sklearn import datasets
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.datasets import make_moons
-from sklearn.tree import export_graphviz
+
+The ID3 algorithm uses this information gain measure to select among the candidate
+attributes at each step while growing the tree.
-Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)
-
-deep_tree_clf1 = DecisionTreeClassifier(random_state=42)
-deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)
-deep_tree_clf1.fit(Xm, ym)
-deep_tree_clf2.fit(Xm, ym)
-
-
-def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):
- x1s = np.linspace(axes[0], axes[1], 100)
- x2s = np.linspace(axes[2], axes[3], 100)
- x1, x2 = np.meshgrid(x1s, x2s)
- X_new = np.c_[x1.ravel(), x2.ravel()]
- y_pred = clf.predict(X_new).reshape(x1.shape)
- custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
- plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
- if not iris:
- custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
- plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
- if plot_training:
- plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa")
- plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor")
- plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica")
- plt.axis(axes)
- if iris:
- plt.xlabel("Petal length", fontsize=14)
- plt.ylabel("Petal width", fontsize=14)
- else:
- plt.xlabel(r"$x_1$", fontsize=18)
- plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
- if legend:
- plt.legend(loc="lower right", fontsize=14)
-plt.figure(figsize=(11, 4))
-plt.subplot(121)
-plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
-plt.title("No restrictions", fontsize=16)
-plt.subplot(122)
-plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
-plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14)
-plt.show()
-
@@ -286,7 +273,7 @@ plt.show()
-
np.random.seed(6)
-Xs = np.random.rand(100, 2) - 0.5
-ys = (Xs[:, 0] > 0).astype(np.float32) * 2
+import re
+import math
+from collections import deque
-angle = np.pi/4
-rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
-Xsr = Xs.dot(rotation_matrix)
+# x is examples in training set
+# y is set of targets
+# label is target attributes
+# Node is a class which has properties values, childs, and next
+# root is top node in the decision tree
-tree_clf_s = DecisionTreeClassifier(random_state=42)
-tree_clf_s.fit(Xs, ys)
-tree_clf_sr = DecisionTreeClassifier(random_state=42)
-tree_clf_sr.fit(Xsr, ys)
+class Node(object):
+ def __init__(self):
+ self.value = None
+ self.next = None
+ self.childs = None
-plt.figure(figsize=(11, 4))
-plt.subplot(121)
-plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
-plt.subplot(122)
-plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+# Simple class of Decision Tree
+# Aimed for who want to learn Decision Tree, so it is not optimized
+class DecisionTree(object):
+ def __init__(self, sample, attributes, labels):
+ self.sample = sample
+ self.attributes = attributes
+ self.labels = labels
+ self.labelCodes = None
+ self.labelCodesCount = None
+ self.initLabelCodes()
+ # print(self.labelCodes)
+ self.root = None
+ self.entropy = self.getEntropy([x for x in range(len(self.labels))])
-plt.show()
+ def initLabelCodes(self):
+ self.labelCodes = []
+ self.labelCodesCount = []
+ for l in self.labels:
+ if l not in self.labelCodes:
+ self.labelCodes.append(l)
+ self.labelCodesCount.append(0)
+ self.labelCodesCount[self.labelCodes.index(l)] += 1
+
+ def getLabelCodeId(self, sampleId):
+ return self.labelCodes.index(self.labels[sampleId])
+
+ def getAttributeValues(self, sampleIds, attributeId):
+ vals = []
+ for sid in sampleIds:
+ val = self.sample[sid][attributeId]
+ if val not in vals:
+ vals.append(val)
+ # print(vals)
+ return vals
+
+ def getEntropy(self, sampleIds):
+ entropy = 0
+ labelCount = [0] * len(self.labelCodes)
+ for sid in sampleIds:
+ labelCount[self.getLabelCodeId(sid)] += 1
+ # print("-ge", labelCount)
+ for lv in labelCount:
+ # print(lv)
+ if lv != 0:
+ entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
+ else:
+ entropy += 0
+ return entropy
+
+ def getDominantLabel(self, sampleIds):
+ labelCodesCount = [0] * len(self.labelCodes)
+ for sid in sampleIds:
+ labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
+ return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
+
+ def getInformationGain(self, sampleIds, attributeId):
+ gain = self.getEntropy(sampleIds)
+ attributeVals = []
+ attributeValsCount = []
+ attributeValsIds = []
+ for sid in sampleIds:
+ val = self.sample[sid][attributeId]
+ if val not in attributeVals:
+ attributeVals.append(val)
+ attributeValsCount.append(0)
+ attributeValsIds.append([])
+ vid = attributeVals.index(val)
+ attributeValsCount[vid] += 1
+ attributeValsIds[vid].append(sid)
+ # print("-gig", self.attributes[attributeId])
+ for vc, vids in zip(attributeValsCount, attributeValsIds):
+ # print("-gig", vids)
+ gain -= vc/len(sampleIds) * self.getEntropy(vids)
+ return gain
+
+ def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
+ attributesEntropy = [0] * len(attributeIds)
+ for i, attId in zip(range(len(attributeIds)), attributeIds):
+ attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
+ maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
+ return self.attributes[maxId], maxId
+
+ def isSingleLabeled(self, sampleIds):
+ label = self.labels[sampleIds[0]]
+ for sid in sampleIds:
+ if self.labels[sid] != label:
+ return False
+ return True
+
+ def getLabel(self, sampleId):
+ return self.labels[sampleId]
+
+ def id3(self):
+ sampleIds = [x for x in range(len(self.sample))]
+ attributeIds = [x for x in range(len(self.attributes))]
+ self.root = self.id3Recv(sampleIds, attributeIds, self.root)
+
+ def id3Recv(self, sampleIds, attributeIds, root):
+ root = Node() # Initialize current root
+ if self.isSingleLabeled(sampleIds):
+ root.value = self.labels[sampleIds[0]]
+ return root
+ # print(attributeIds)
+ if len(attributeIds) == 0:
+ root.value = self.getDominantLabel(sampleIds)
+ return root
+ bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
+ sampleIds, attributeIds)
+ # print(bestAttrName)
+ root.value = bestAttrName
+ root.childs = [] # Create list of children
+ for value in self.getAttributeValues(sampleIds, bestAttrId):
+ # print(value)
+ child = Node()
+ child.value = value
+ root.childs.append(child) # Append new child node to current
+ # root
+ childSampleIds = []
+ for sid in sampleIds:
+ if self.sample[sid][bestAttrId] == value:
+ childSampleIds.append(sid)
+ if len(childSampleIds) == 0:
+ child.next = self.getDominantLabel(sampleIds)
+ else:
+ # print(bestAttrName, bestAttrId)
+ # print(attributeIds)
+ if len(attributeIds) > 0 and bestAttrId in attributeIds:
+ toRemove = attributeIds.index(bestAttrId)
+ attributeIds.pop(toRemove)
+ child.next = self.id3Recv(
+ childSampleIds, attributeIds, child.next)
+ return root
+
+ def printTree(self):
+ if self.root:
+ roots = deque()
+ roots.append(self.root)
+ while len(roots) > 0:
+ root = roots.popleft()
+ print(root.value)
+ if root.childs:
+ for child in root.childs:
+ print('({})'.format(child.value))
+ roots.append(child.next)
+ elif root.next:
+ print(root.next)
+
+
+def test():
+ f = open('DataFiles/rideclass.csv')
+ attributes = f.readline().split(',')
+ attributes = attributes[1:len(attributes)-1]
+ print(attributes)
+ sample = f.readlines()
+ f.close()
+ for i in range(len(sample)):
+ sample[i] = re.sub('\d+,', '', sample[i])
+ sample[i] = sample[i].strip().split(',')
+ labels = []
+ for s in sample:
+ labels.append(s.pop())
+ # print(sample)
+ # print(labels)
+ decisionTree = DecisionTree(sample, attributes, labels)
+ print("System entropy {}".format(decisionTree.entropy))
+ decisionTree.id3()
+ decisionTree.printTree()
+
+
+if __name__ == '__main__':
+ test()
@@ -242,7 +433,7 @@ plt.show()
-
# Quadratic training set + noise
-np.random.seed(42)
-m = 200
-X = np.random.rand(m, 1)
-y = 4 * (X - 0.5) ** 2
-y = y + np.random.randn(m, 1) / 10
-+
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+from sklearn.svm import SVC
+from sklearn.linear_model import LogisticRegression
+from sklearn.tree import DecisionTreeClassifier
-
-from sklearn.tree import DecisionTreeRegressor
+# Load the data
+cancer = load_breast_cancer()
-tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg.fit(X, y)
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+# Logistic Regression
+logreg = LogisticRegression(solver='lbfgs')
+logreg.fit(X_train, y_train)
+print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
+# Support vector machine
+svm = SVC(gamma='auto', C=100)
+svm.fit(X_train, y_train)
+print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
+# Decision Trees
+deep_tree_clf = DecisionTreeClassifier(max_depth=None)
+deep_tree_clf.fit(X_train, y_train)
+print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+# Logistic Regression
+logreg.fit(X_train_scaled, y_train)
+print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Support Vector Machine
+svm.fit(X_train_scaled, y_train)
+print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Decision Trees
+deep_tree_clf.fit(X_train_scaled, y_train)
+print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
@@ -236,7 +286,7 @@ tree_reg.fit(X, y)
-
from sklearn.tree import DecisionTreeRegressor
+from __future__ import division, print_function, unicode_literals
-tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)
-tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)
-tree_reg1.fit(X, y)
-tree_reg2.fit(X, y)
+# Common imports
+import numpy as np
+import os
-def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel="$y$"):
- x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)
- y_pred = tree_reg.predict(x1)
- plt.axis(axes)
- plt.xlabel("$x_1$", fontsize=18)
- if ylabel:
- plt.ylabel(ylabel, fontsize=18, rotation=0)
- plt.plot(X, y, "b.")
- plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$")
+# to make this notebook's output stable across runs
+np.random.seed(42)
+# To plot pretty figures
+import matplotlib
+import matplotlib.pyplot as plt
+from matplotlib.colors import ListedColormap
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+from sklearn.svm import SVC
+from sklearn import datasets
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.datasets import make_moons
+from sklearn.tree import export_graphviz
+
+Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)
+
+deep_tree_clf1 = DecisionTreeClassifier(random_state=42)
+deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)
+deep_tree_clf1.fit(Xm, ym)
+deep_tree_clf2.fit(Xm, ym)
+
+
+def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):
+ x1s = np.linspace(axes[0], axes[1], 100)
+ x2s = np.linspace(axes[2], axes[3], 100)
+ x1, x2 = np.meshgrid(x1s, x2s)
+ X_new = np.c_[x1.ravel(), x2.ravel()]
+ y_pred = clf.predict(X_new).reshape(x1.shape)
+ custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
+ plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
+ if not iris:
+ custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
+ plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
+ if plot_training:
+ plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa")
+ plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor")
+ plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica")
+ plt.axis(axes)
+ if iris:
+ plt.xlabel("Petal length", fontsize=14)
+ plt.ylabel("Petal width", fontsize=14)
+ else:
+ plt.xlabel(r"$x_1$", fontsize=18)
+ plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
+ if legend:
+ plt.legend(loc="lower right", fontsize=14)
plt.figure(figsize=(11, 4))
plt.subplot(121)
-plot_regression_predictions(tree_reg1, X, y)
-for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
- plt.plot([split, split], [-0.2, 1], style, linewidth=2)
-plt.text(0.21, 0.65, "Depth=0", fontsize=15)
-plt.text(0.01, 0.2, "Depth=1", fontsize=13)
-plt.text(0.65, 0.8, "Depth=1", fontsize=13)
-plt.legend(loc="upper center", fontsize=18)
-plt.title("max_depth=2", fontsize=14)
-
+plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
+plt.title("No restrictions", fontsize=16)
plt.subplot(122)
-plot_regression_predictions(tree_reg2, X, y, ylabel=None)
-for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
- plt.plot([split, split], [-0.2, 1], style, linewidth=2)
-for split in (0.0458, 0.1298, 0.2873, 0.9040):
- plt.plot([split, split], [-0.2, 1], "k:", linewidth=1)
-plt.text(0.3, 0.5, "Depth=2", fontsize=13)
-plt.title("max_depth=3", fontsize=14)
-
-plt.show()
-
-
-
-
-
tree_reg1 = DecisionTreeRegressor(random_state=42)
-tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)
-tree_reg1.fit(X, y)
-tree_reg2.fit(X, y)
-
-x1 = np.linspace(0, 1, 500).reshape(-1, 1)
-y_pred1 = tree_reg1.predict(x1)
-y_pred2 = tree_reg2.predict(x1)
-
-plt.figure(figsize=(11, 4))
-
-plt.subplot(121)
-plt.plot(X, y, "b.")
-plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$")
-plt.axis([0, 1, -0.2, 1.1])
-plt.xlabel("$x_1$", fontsize=18)
-plt.ylabel("$y$", fontsize=18, rotation=0)
-plt.legend(loc="upper center", fontsize=18)
-plt.title("No restrictions", fontsize=14)
-
-plt.subplot(122)
-plt.plot(X, y, "b.")
-plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$")
-plt.axis([0, 1, -0.2, 1.1])
-plt.xlabel("$x_1$", fontsize=18)
-plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14)
-
+plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
+plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14)
plt.show()
@@ -292,7 +309,7 @@ plt.show()
-
np.random.seed(6)
+Xs = np.random.rand(100, 2) - 0.5
+ys = (Xs[:, 0] > 0).astype(np.float32) * 2
+angle = np.pi/4
+rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
+Xsr = Xs.dot(rotation_matrix)
+
+tree_clf_s = DecisionTreeClassifier(random_state=42)
+tree_clf_s.fit(Xs, ys)
+tree_clf_sr = DecisionTreeClassifier(random_state=42)
+tree_clf_sr.fit(Xsr, ys)
+
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+plt.subplot(122)
+plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+
+plt.show()
+
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs028.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs028.html index 830923076..767d35e1c 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs028.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs028.html @@ -64,40 +64,55 @@ Automatically generated HTML file from DocOnce source ('Classification tree, how to split nodes', 2, None, '___sec13'), ('Visualizing the Tree, Classification', 2, None, '___sec14'), ('Visualizing the Tree, The Moons', 2, None, '___sec15'), - ('Computing the Gini index', 2, None, '___sec16'), + ('Algorithms for Setting up Decision Trees', 2, None, '___sec16'), + ('The CART algorithm for Classification', 2, None, '___sec17'), + ('The CART algorithm for Regression', 2, None, '___sec18'), + ('Computing the Gini index', 2, None, '___sec19'), ('Simple Python Code to read in Data and perform Classification', 2, None, - '___sec17'), - ('Computing the Gini Factor', 2, None, '___sec18'), - ('Entropy and the ID3 algorithm', 2, None, '___sec19'), - ('Implementing the ID3 Algorithm', 2, None, '___sec20'), + '___sec20'), + ('Computing the Gini Factor', 2, None, '___sec21'), + ('Entropy and the ID3 algorithm', 2, None, '___sec22'), + ('Implementing the ID3 Algorithm', 2, None, '___sec23'), ('Cancer Data again now with Decision Trees and other Methods', 2, None, - '___sec21'), - ('Another example, the moons again', 2, None, '___sec22'), - ('Playing around with regions', 2, None, '___sec23'), - ('Regression trees', 2, None, '___sec24'), - ('Final regressor code', 2, None, '___sec25'), - ('Pros and cons of trees, pros', 2, None, '___sec26'), - ('Disadvantages', 2, None, '___sec27'), - ('Bagging', 2, None, '___sec28'), - ('More bagging', 2, None, '___sec29'), - ('Simple Voting Example, head or tail', 2, None, '___sec30'), - ('Using the Voting Classifier', 2, None, '___sec31'), + '___sec24'), + ('Another example, the moons again', 2, None, '___sec25'), + ('Playing around with regions', 2, None, '___sec26'), + ('Regression trees', 2, None, '___sec27'), + ('Final regressor code', 2, None, '___sec28'), + ('Pros and cons of trees, pros', 2, None, '___sec29'), + ('Disadvantages', 2, None, '___sec30'), + ('From a Single Tree to Many Trees, that is meet the Jungle of ' + 'Methods', + 2, + None, + '___sec31'), + ('Bagging', 2, None, '___sec32'), + ('More bagging', 2, None, '___sec33'), + ('Simple Voting Example, head or tail', 2, None, '___sec34'), + ('Using the Voting Classifier', 2, None, '___sec35'), ('Please, not the moons again! Voting and Bagging', 2, None, - '___sec32'), - ('Now Bagging', 2, None, '___sec33'), - ('Random forests', 2, None, '___sec34'), - ('A simple scikit-learn example', 2, None, '___sec35'), - ('Then random forests', 2, None, '___sec36'), - ('Feature Importance', 2, None, '___sec37'), - ('Boosting: AdaBoost', 2, None, '___sec38'), - ('Gradient Boosting', 2, None, '___sec39'), - ('Gradient Boots with Early Stopping', 2, None, '___sec40')]} + '___sec36'), + ('Now Bagging', 2, None, '___sec37'), + ('Random forests', 2, None, '___sec38'), + ('A simple scikit-learn example', 2, None, '___sec39'), + ('Then random forests', 2, None, '___sec40'), + ('Feature Importance', 2, None, '___sec41'), + ("Boosting, a Bird'e Eye", 2, None, '___sec42'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + '___sec43'), + ('AdaBoost Examples', 2, None, '___sec44'), + ('Gradient boosting: Basic Algorithm', 2, None, '___sec45'), + ('Gradient Boosting, Examples', 2, None, '___sec46'), + ('Gradient Boots with Early Stopping', 2, None, '___sec47'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec48')]} end of tocinfo --> @@ -151,31 +166,39 @@ MathJax.Hub.Config({
-
# Quadratic training set + noise
+np.random.seed(42)
+m = 200
+X = np.random.rand(m, 1)
+y = 4 * (X - 0.5) ** 2
+y = y + np.random.randn(m, 1) / 10
+-However, by aggregating many decision trees, using methods like bagging, random forests, and boosting, the predictive performance of trees can be substantially improved. + +
from sklearn.tree import DecisionTreeRegressor
+tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg.fit(X, y)
+
@@ -231,7 +259,7 @@ However, by aggregating many decision trees, using methods like bagging, random
-The plain decision trees suffer from high -variance. This means that if we split the training data into two parts -at random, and fit a decision tree to both halves, the results that we -get could be quite different. In contrast, a procedure with low -variance will yield similar results if applied repeatedly to distinct -data sets; linear regression tends to have low variance, if the ratio -of \( n \) to \( p \) is moderately large. + +
from sklearn.tree import DecisionTreeRegressor
+
+tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)
+tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)
+tree_reg1.fit(X, y)
+tree_reg2.fit(X, y)
+
+def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel="$y$"):
+ x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)
+ y_pred = tree_reg.predict(x1)
+ plt.axis(axes)
+ plt.xlabel("$x_1$", fontsize=18)
+ if ylabel:
+ plt.ylabel(ylabel, fontsize=18, rotation=0)
+ plt.plot(X, y, "b.")
+ plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$")
+
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_regression_predictions(tree_reg1, X, y)
+for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
+ plt.plot([split, split], [-0.2, 1], style, linewidth=2)
+plt.text(0.21, 0.65, "Depth=0", fontsize=15)
+plt.text(0.01, 0.2, "Depth=1", fontsize=13)
+plt.text(0.65, 0.8, "Depth=1", fontsize=13)
+plt.legend(loc="upper center", fontsize=18)
+plt.title("max_depth=2", fontsize=14)
+
+plt.subplot(122)
+plot_regression_predictions(tree_reg2, X, y, ylabel=None)
+for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
+ plt.plot([split, split], [-0.2, 1], style, linewidth=2)
+for split in (0.0458, 0.1298, 0.2873, 0.9040):
+ plt.plot([split, split], [-0.2, 1], "k:", linewidth=1)
+plt.text(0.3, 0.5, "Depth=2", fontsize=13)
+plt.title("max_depth=3", fontsize=14)
+
+plt.show()
+-Bootstrap aggregation, or just bagging, is a -general-purpose procedure for reducing the variance of a statistical -learning method. + +
tree_reg1 = DecisionTreeRegressor(random_state=42)
+tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)
+tree_reg1.fit(X, y)
+tree_reg2.fit(X, y)
+
+x1 = np.linspace(0, 1, 500).reshape(-1, 1)
+y_pred1 = tree_reg1.predict(x1)
+y_pred2 = tree_reg2.predict(x1)
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.plot(X, y, "b.")
+plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$")
+plt.axis([0, 1, -0.2, 1.1])
+plt.xlabel("$x_1$", fontsize=18)
+plt.ylabel("$y$", fontsize=18, rotation=0)
+plt.legend(loc="upper center", fontsize=18)
+plt.title("No restrictions", fontsize=14)
+
+plt.subplot(122)
+plt.plot(X, y, "b.")
+plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$")
+plt.axis([0, 1, -0.2, 1.1])
+plt.xlabel("$x_1$", fontsize=18)
+plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14)
+
+plt.show()
+
@@ -233,7 +315,7 @@ learning method.
-Bagging typically results in improved accuracy -over prediction using a single tree. Unfortunately, however, it can be -difficult to interpret the resulting model. Recall that one of the -advantages of decision trees is the attractive and easily interpreted -diagram that results. +
-However, when we bag a large number of trees, it is no longer -possible to represent the resulting statistical learning procedure -using a single tree, and it is no longer clear which variables are -most important to the procedure. Thus, bagging improves prediction -accuracy at the expense of interpretability. Although the collection -of bagged trees is much more difficult to interpret than a single -tree, one can obtain an overall summary of the importance of each -predictor using the MSE (for bagging regression trees) or the Gini -index (for bagging classification trees). In the case of bagging -regression trees, we can record the total amount that the MSE is -decreased due to splits over a given predictor, averaged over all \( B \) possible -trees. A large value indicates an important predictor. Similarly, in -the context of bagging classification trees, we can add up the total -amount that the Gini index is decreased by splits over a given -predictor, averaged over all \( B \) trees. - -
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs031.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs031.html index 044ef068d..21e98113b 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs031.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs031.html @@ -64,40 +64,55 @@ Automatically generated HTML file from DocOnce source ('Classification tree, how to split nodes', 2, None, '___sec13'), ('Visualizing the Tree, Classification', 2, None, '___sec14'), ('Visualizing the Tree, The Moons', 2, None, '___sec15'), - ('Computing the Gini index', 2, None, '___sec16'), + ('Algorithms for Setting up Decision Trees', 2, None, '___sec16'), + ('The CART algorithm for Classification', 2, None, '___sec17'), + ('The CART algorithm for Regression', 2, None, '___sec18'), + ('Computing the Gini index', 2, None, '___sec19'), ('Simple Python Code to read in Data and perform Classification', 2, None, - '___sec17'), - ('Computing the Gini Factor', 2, None, '___sec18'), - ('Entropy and the ID3 algorithm', 2, None, '___sec19'), - ('Implementing the ID3 Algorithm', 2, None, '___sec20'), + '___sec20'), + ('Computing the Gini Factor', 2, None, '___sec21'), + ('Entropy and the ID3 algorithm', 2, None, '___sec22'), + ('Implementing the ID3 Algorithm', 2, None, '___sec23'), ('Cancer Data again now with Decision Trees and other Methods', 2, None, - '___sec21'), - ('Another example, the moons again', 2, None, '___sec22'), - ('Playing around with regions', 2, None, '___sec23'), - ('Regression trees', 2, None, '___sec24'), - ('Final regressor code', 2, None, '___sec25'), - ('Pros and cons of trees, pros', 2, None, '___sec26'), - ('Disadvantages', 2, None, '___sec27'), - ('Bagging', 2, None, '___sec28'), - ('More bagging', 2, None, '___sec29'), - ('Simple Voting Example, head or tail', 2, None, '___sec30'), - ('Using the Voting Classifier', 2, None, '___sec31'), + '___sec24'), + ('Another example, the moons again', 2, None, '___sec25'), + ('Playing around with regions', 2, None, '___sec26'), + ('Regression trees', 2, None, '___sec27'), + ('Final regressor code', 2, None, '___sec28'), + ('Pros and cons of trees, pros', 2, None, '___sec29'), + ('Disadvantages', 2, None, '___sec30'), + ('From a Single Tree to Many Trees, that is meet the Jungle of ' + 'Methods', + 2, + None, + '___sec31'), + ('Bagging', 2, None, '___sec32'), + ('More bagging', 2, None, '___sec33'), + ('Simple Voting Example, head or tail', 2, None, '___sec34'), + ('Using the Voting Classifier', 2, None, '___sec35'), ('Please, not the moons again! Voting and Bagging', 2, None, - '___sec32'), - ('Now Bagging', 2, None, '___sec33'), - ('Random forests', 2, None, '___sec34'), - ('A simple scikit-learn example', 2, None, '___sec35'), - ('Then random forests', 2, None, '___sec36'), - ('Feature Importance', 2, None, '___sec37'), - ('Boosting: AdaBoost', 2, None, '___sec38'), - ('Gradient Boosting', 2, None, '___sec39'), - ('Gradient Boots with Early Stopping', 2, None, '___sec40')]} + '___sec36'), + ('Now Bagging', 2, None, '___sec37'), + ('Random forests', 2, None, '___sec38'), + ('A simple scikit-learn example', 2, None, '___sec39'), + ('Then random forests', 2, None, '___sec40'), + ('Feature Importance', 2, None, '___sec41'), + ("Boosting, a Bird'e Eye", 2, None, '___sec42'), + ('Adaptive boosting: AdaBoost, Basic Algorithm', + 2, + None, + '___sec43'), + ('AdaBoost Examples', 2, None, '___sec44'), + ('Gradient boosting: Basic Algorithm', 2, None, '___sec45'), + ('Gradient Boosting, Examples', 2, None, '___sec46'), + ('Gradient Boots with Early Stopping', 2, None, '___sec47'), + ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec48')]} end of tocinfo --> @@ -151,31 +166,39 @@ MathJax.Hub.Config({
+
heads_proba = 0.51
-coin_tosses = (np.random.rand(10000, 10) < heads_proba).astype(np.int32)
-cumulative_heads_ratio = np.cumsum(coin_tosses, axis=0) / np.arange(1, 10001).reshape(-1, 1)
-plt.figure(figsize=(8,3.5))
-plt.plot(cumulative_heads_ratio)
-plt.plot([0, 10000], [0.51, 0.51], "k--", linewidth=2, label="51%")
-plt.plot([0, 10000], [0.5, 0.5], "k-", label="50%")
-plt.xlabel("Number of coin tosses")
-plt.ylabel("Heads ratio")
-plt.legend(loc="lower right")
-plt.axis([0, 10000, 0.42, 0.58])
-plt.show()
-
@@ -234,7 +254,7 @@ plt.show()
+As stated above and seen in many of the examples discussed here about +a single decision tree, we often end up overfitting our training +data. This normally means that we have a high variance. Can we reduce +the variance of a statistical learning method? - -
from sklearn.model_selection import train_test_split
-from sklearn.datasets import make_moons
+
+This leads us to a set of different methods that can combine different
+machine learning algorithms or just use one of them to construct forests and jungles of trees, homogeneous ones or heterogenous ones. These methods are recognized by different names which we will try to explain here. These are
-X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
-X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
+
+- Votign classifiers
+- Bagging and Pasting
+- Random forests
+- Boosting methods
+
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.ensemble import VotingClassifier
-from sklearn.linear_model import LogisticRegression
-from sklearn.svm import SVC
+We discuss these methods here.
-log_clf = LogisticRegression(solver="liblinear", random_state=42)
-rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
-svm_clf = SVC(gamma="auto", random_state=42)
-
-voting_clf = VotingClassifier(
- estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
- voting='hard')
-
-voting_clf.fit(X_train, y_train)
-
-from sklearn.metrics import accuracy_score
-
-for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
- clf.fit(X_train, y_train)
- y_pred = clf.predict(X_test)
- print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
-
-log_clf = LogisticRegression(solver="liblinear", random_state=42)
-rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
-svm_clf = SVC(gamma="auto", probability=True, random_state=42)
-
-voting_clf = VotingClassifier(
- estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
- voting='soft')
-voting_clf.fit(X_train, y_train)
-
-from sklearn.metrics import accuracy_score
-
-for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
- clf.fit(X_train, y_train)
- y_pred = clf.predict(X_test)
- print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
-
@@ -264,6 +260,8 @@ voting_clf.fit(X_train, y_train)
+The plain decision trees suffer from high +variance. This means that if we split the training data into two parts +at random, and fit a decision tree to both halves, the results that we +get could be quite different. In contrast, a procedure with low +variance will yield similar results if applied repeatedly to distinct +data sets; linear regression tends to have low variance, if the ratio +of \( n \) to \( p \) is moderately large. - -
from sklearn.model_selection import train_test_split
-from sklearn.datasets import make_moons
-
-X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
-X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.ensemble import VotingClassifier
-from sklearn.linear_model import LogisticRegression
-from sklearn.svm import SVC
-
-log_clf = LogisticRegression(random_state=42)
-rnd_clf = RandomForestClassifier(random_state=42)
-svm_clf = SVC(random_state=42)
-
-voting_clf = VotingClassifier(
- estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
- voting='hard')
-voting_clf.fit(X_train, y_train)
-+Bootstrap aggregation, or just bagging, is a +general-purpose procedure for reducing the variance of a statistical +learning method. - -
from sklearn.metrics import accuracy_score
-
-for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
- clf.fit(X_train, y_train)
- y_pred = clf.predict(X_test)
- print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
-- - -
log_clf = LogisticRegression(random_state=42)
-rnd_clf = RandomForestClassifier(random_state=42)
-svm_clf = SVC(probability=True, random_state=42)
-
-voting_clf = VotingClassifier(
- estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
- voting='soft')
-voting_clf.fit(X_train, y_train)
-- - -
from sklearn.metrics import accuracy_score
-
-for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
- clf.fit(X_train, y_train)
- y_pred = clf.predict(X_test)
- print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
-
@@ -270,6 +254,9 @@ voting_clf.fit(X_train, y_train)
+Bagging typically results in improved accuracy +over prediction using a single tree. Unfortunately, however, it can be +difficult to interpret the resulting model. Recall that one of the +advantages of decision trees is the attractive and easily interpreted +diagram that results. - -
from sklearn.ensemble import BaggingClassifier
-from sklearn.tree import DecisionTreeClassifier
-
-bag_clf = BaggingClassifier(
- DecisionTreeClassifier(random_state=42), n_estimators=500,
- max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)
-bag_clf.fit(X_train, y_train)
-y_pred = bag_clf.predict(X_test)
-+However, when we bag a large number of trees, it is no longer +possible to represent the resulting statistical learning procedure +using a single tree, and it is no longer clear which variables are +most important to the procedure. Thus, bagging improves prediction +accuracy at the expense of interpretability. Although the collection +of bagged trees is much more difficult to interpret than a single +tree, one can obtain an overall summary of the importance of each +predictor using the MSE (for bagging regression trees) or the Gini +index (for bagging classification trees). In the case of bagging +regression trees, we can record the total amount that the MSE is +decreased due to splits over a given predictor, averaged over all \( B \) possible +trees. A large value indicates an important predictor. Similarly, in +the context of bagging classification trees, we can add up the total +amount that the Gini index is decreased by splits over a given +predictor, averaged over all \( B \) trees. - -
from sklearn.metrics import accuracy_score
-print(accuracy_score(y_test, y_pred))
-- - -
tree_clf = DecisionTreeClassifier(random_state=42)
-tree_clf.fit(X_train, y_train)
-y_pred_tree = tree_clf.predict(X_test)
-print(accuracy_score(y_test, y_pred_tree))
-- - -
from matplotlib.colors import ListedColormap
-
-def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):
- x1s = np.linspace(axes[0], axes[1], 100)
- x2s = np.linspace(axes[2], axes[3], 100)
- x1, x2 = np.meshgrid(x1s, x2s)
- X_new = np.c_[x1.ravel(), x2.ravel()]
- y_pred = clf.predict(X_new).reshape(x1.shape)
- custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
- plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
- if contour:
- custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
- plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
- plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha)
- plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha)
- plt.axis(axes)
- plt.xlabel(r"$x_1$", fontsize=18)
- plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
-plt.figure(figsize=(11,4))
-plt.subplot(121)
-plot_decision_boundary(tree_clf, X, y)
-plt.title("Decision Tree", fontsize=14)
-plt.subplot(122)
-plot_decision_boundary(bag_clf, X, y)
-plt.title("Decision Trees with Bagging", fontsize=14)
-plt.show()
-
@@ -272,6 +263,10 @@ plt.show()
-Random forests provide an improvement over bagged trees by way of a -small tweak that decorrelates the trees. - -
-As in bagging, we build a -number of decision trees on bootstrapped training samples. But when -building these decision trees, each time a split in a tree is -considered, a random sample of \( m \) predictors is chosen as split -candidates from the full set of \( p \) predictors. The split is allowed to -use only one of those \( m \) predictors. - -
-A fresh sample of \( m \) predictors is -taken at each split, and typically we choose - -$$ -m\approx \sqrt{p}. -$$ - -
-In building a random forest, at -each split in the tree, the algorithm is not even allowed to consider -a majority of the available predictors. - -
-The reason for this is rather clever. Suppose that there is one very -strong predictor in the data set, along with a number of other -moderately strong predictors. Then in the collection of bagged -variable importance random forest trees, most or all of the trees will -use this strong predictor in the top split. Consequently, all of the -bagged trees will look quite similar to each other. Hence the -predictions from the bagged trees will be highly correlated. -Unfortunately, averaging many highly correlated quantities does not -lead to as large of a reduction in variance as averaging many -uncorrelated quanti- ties. In particular, this means that bagging will -not lead to a substantial reduction in variance over a single tree in -this setting. + +
heads_proba = 0.51
+coin_tosses = (np.random.rand(10000, 10) < heads_proba).astype(np.int32)
+cumulative_heads_ratio = np.cumsum(coin_tosses, axis=0) / np.arange(1, 10001).reshape(-1, 1)
+plt.figure(figsize=(8,3.5))
+plt.plot(cumulative_heads_ratio)
+plt.plot([0, 10000], [0.51, 0.51], "k--", linewidth=2, label="51%")
+plt.plot([0, 10000], [0.5, 0.5], "k-", label="50%")
+plt.xlabel("Number of coin tosses")
+plt.ylabel("Heads ratio")
+plt.legend(loc="lower right")
+plt.axis([0, 10000, 0.42, 0.58])
+plt.show()
+
@@ -254,6 +253,11 @@ this setting.
-
from sklearn.ensemble import RandomForestClassifier
-from sklearn.preprocessing import LabelEncoder
-from sklearn.model_selection import cross_validate
-# Data set not specificied
-X = dataset.XXX
-Y = dataset.YYY
-#Instantiate the model with 100 trees and entropy as splitting criteria
-Random_Forest_model = RandomForestClassifier(n_estimators=100,criterion="entropy")
-#Cross validation
-accuracy = cross_validate(Random_Forest_model,X,Y,cv=10)['test_score']
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import make_moons
+
+X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
+X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
+
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.ensemble import VotingClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.svm import SVC
+
+log_clf = LogisticRegression(solver="liblinear", random_state=42)
+rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
+svm_clf = SVC(gamma="auto", random_state=42)
+
+voting_clf = VotingClassifier(
+ estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
+ voting='hard')
+
+voting_clf.fit(X_train, y_train)
+
+from sklearn.metrics import accuracy_score
+
+for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
+ clf.fit(X_train, y_train)
+ y_pred = clf.predict(X_test)
+ print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
+
+log_clf = LogisticRegression(solver="liblinear", random_state=42)
+rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
+svm_clf = SVC(gamma="auto", probability=True, random_state=42)
+
+voting_clf = VotingClassifier(
+ estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
+ voting='soft')
+voting_clf.fit(X_train, y_train)
+
+from sklearn.metrics import accuracy_score
+
+for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
+ clf.fit(X_train, y_train)
+ y_pred = clf.predict(X_test)
+ print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
@@ -227,6 +283,12 @@ accuracy = cross_validate(Random_Forest_mode
-
bag_clf = BaggingClassifier(
- DecisionTreeClassifier(splitter="random", max_leaf_nodes=16, random_state=42),
- n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1, random_state=42)
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import make_moons
+
+X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
+X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.ensemble import VotingClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.svm import SVC
+
+log_clf = LogisticRegression(random_state=42)
+rnd_clf = RandomForestClassifier(random_state=42)
+svm_clf = SVC(random_state=42)
+
+voting_clf = VotingClassifier(
+ estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
+ voting='hard')
+voting_clf.fit(X_train, y_train)
-
bag_clf.fit(X_train, y_train)
-y_pred = bag_clf.predict(X_test)
-from sklearn.ensemble import RandomForestClassifier
-rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)
-rnd_clf.fit(X_train, y_train)
-y_pred_rf = rnd_clf.predict(X_test)
-np.sum(y_pred == y_pred_rf) / len(y_pred)
+from sklearn.metrics import accuracy_score
+
+for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
+ clf.fit(X_train, y_train)
+ y_pred = clf.predict(X_test)
+ print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
+
+
+
+
+
log_clf = LogisticRegression(random_state=42)
+rnd_clf = RandomForestClassifier(random_state=42)
+svm_clf = SVC(probability=True, random_state=42)
+
+voting_clf = VotingClassifier(
+ estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
+ voting='soft')
+voting_clf.fit(X_train, y_train)
+
+
+
+
+
from sklearn.metrics import accuracy_score
+
+for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
+ clf.fit(X_train, y_train)
+ y_pred = clf.predict(X_test)
+ print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
@@ -230,6 +289,13 @@ np.sum(y_pred =
40
41
42
+ 43
+ 44
+ 45
+ 46
+ 47
+ ...
+ 50
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs038.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs038.html
index 6f47e526d..cadb27ef5 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs038.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs038.html
@@ -64,40 +64,55 @@ Automatically generated HTML file from DocOnce source
('Classification tree, how to split nodes', 2, None, '___sec13'),
('Visualizing the Tree, Classification', 2, None, '___sec14'),
('Visualizing the Tree, The Moons', 2, None, '___sec15'),
- ('Computing the Gini index', 2, None, '___sec16'),
+ ('Algorithms for Setting up Decision Trees', 2, None, '___sec16'),
+ ('The CART algorithm for Classification', 2, None, '___sec17'),
+ ('The CART algorithm for Regression', 2, None, '___sec18'),
+ ('Computing the Gini index', 2, None, '___sec19'),
('Simple Python Code to read in Data and perform Classification',
2,
None,
- '___sec17'),
- ('Computing the Gini Factor', 2, None, '___sec18'),
- ('Entropy and the ID3 algorithm', 2, None, '___sec19'),
- ('Implementing the ID3 Algorithm', 2, None, '___sec20'),
+ '___sec20'),
+ ('Computing the Gini Factor', 2, None, '___sec21'),
+ ('Entropy and the ID3 algorithm', 2, None, '___sec22'),
+ ('Implementing the ID3 Algorithm', 2, None, '___sec23'),
('Cancer Data again now with Decision Trees and other Methods',
2,
None,
- '___sec21'),
- ('Another example, the moons again', 2, None, '___sec22'),
- ('Playing around with regions', 2, None, '___sec23'),
- ('Regression trees', 2, None, '___sec24'),
- ('Final regressor code', 2, None, '___sec25'),
- ('Pros and cons of trees, pros', 2, None, '___sec26'),
- ('Disadvantages', 2, None, '___sec27'),
- ('Bagging', 2, None, '___sec28'),
- ('More bagging', 2, None, '___sec29'),
- ('Simple Voting Example, head or tail', 2, None, '___sec30'),
- ('Using the Voting Classifier', 2, None, '___sec31'),
+ '___sec24'),
+ ('Another example, the moons again', 2, None, '___sec25'),
+ ('Playing around with regions', 2, None, '___sec26'),
+ ('Regression trees', 2, None, '___sec27'),
+ ('Final regressor code', 2, None, '___sec28'),
+ ('Pros and cons of trees, pros', 2, None, '___sec29'),
+ ('Disadvantages', 2, None, '___sec30'),
+ ('From a Single Tree to Many Trees, that is meet the Jungle of '
+ 'Methods',
+ 2,
+ None,
+ '___sec31'),
+ ('Bagging', 2, None, '___sec32'),
+ ('More bagging', 2, None, '___sec33'),
+ ('Simple Voting Example, head or tail', 2, None, '___sec34'),
+ ('Using the Voting Classifier', 2, None, '___sec35'),
('Please, not the moons again! Voting and Bagging',
2,
None,
- '___sec32'),
- ('Now Bagging', 2, None, '___sec33'),
- ('Random forests', 2, None, '___sec34'),
- ('A simple scikit-learn example', 2, None, '___sec35'),
- ('Then random forests', 2, None, '___sec36'),
- ('Feature Importance', 2, None, '___sec37'),
- ('Boosting: AdaBoost', 2, None, '___sec38'),
- ('Gradient Boosting', 2, None, '___sec39'),
- ('Gradient Boots with Early Stopping', 2, None, '___sec40')]}
+ '___sec36'),
+ ('Now Bagging', 2, None, '___sec37'),
+ ('Random forests', 2, None, '___sec38'),
+ ('A simple scikit-learn example', 2, None, '___sec39'),
+ ('Then random forests', 2, None, '___sec40'),
+ ('Feature Importance', 2, None, '___sec41'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec42'),
+ ('Adaptive boosting: AdaBoost, Basic Algorithm',
+ 2,
+ None,
+ '___sec43'),
+ ('AdaBoost Examples', 2, None, '___sec44'),
+ ('Gradient boosting: Basic Algorithm', 2, None, '___sec45'),
+ ('Gradient Boosting, Examples', 2, None, '___sec46'),
+ ('Gradient Boots with Early Stopping', 2, None, '___sec47'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec48')]}
end of tocinfo -->
@@ -151,31 +166,39 @@ MathJax.Hub.Config({
Classification tree, how to split nodes
Visualizing the Tree, Classification
Visualizing the Tree, The Moons
- Computing the Gini index
- Simple Python Code to read in Data and perform Classification
- Computing the Gini Factor
- Entropy and the ID3 algorithm
- Implementing the ID3 Algorithm
- Cancer Data again now with Decision Trees and other Methods
- Another example, the moons again
- Playing around with regions
- Regression trees
- Final regressor code
- Pros and cons of trees, pros
- Disadvantages
- Bagging
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier
- Please, not the moons again! Voting and Bagging
- Now Bagging
- Random forests
- A simple scikit-learn example
- Then random forests
- Feature Importance
- Boosting: AdaBoost
- Gradient Boosting
- Gradient Boots with Early Stopping
+ Algorithms for Setting up Decision Trees
+ The CART algorithm for Classification
+ The CART algorithm for Regression
+ Computing the Gini index
+ Simple Python Code to read in Data and perform Classification
+ Computing the Gini Factor
+ Entropy and the ID3 algorithm
+ Implementing the ID3 Algorithm
+ Cancer Data again now with Decision Trees and other Methods
+ Another example, the moons again
+ Playing around with regions
+ Regression trees
+ Final regressor code
+ Pros and cons of trees, pros
+ Disadvantages
+ From a Single Tree to Many Trees, that is meet the Jungle of Methods
+ Bagging
+ More bagging
+ Simple Voting Example, head or tail
+ Using the Voting Classifier
+ Please, not the moons again! Voting and Bagging
+ Now Bagging
+ Random forests
+ A simple scikit-learn example
+ Then random forests
+ Feature Importance
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ AdaBoost Examples
+ Gradient boosting: Basic Algorithm
+ Gradient Boosting, Examples
+ Gradient Boots with Early Stopping
+ XGBoost: Extreme Gradient Boosting
@@ -191,11 +214,64 @@ MathJax.Hub.Config({
-Feature Importance
+Now Bagging
-Example will be added here.
+
+
from sklearn.ensemble import BaggingClassifier
+from sklearn.tree import DecisionTreeClassifier
+
+bag_clf = BaggingClassifier(
+ DecisionTreeClassifier(random_state=42), n_estimators=500,
+ max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)
+bag_clf.fit(X_train, y_train)
+y_pred = bag_clf.predict(X_test)
+
+
+
+
+
from sklearn.metrics import accuracy_score
+print(accuracy_score(y_test, y_pred))
+
+
+
+
+
tree_clf = DecisionTreeClassifier(random_state=42)
+tree_clf.fit(X_train, y_train)
+y_pred_tree = tree_clf.predict(X_test)
+print(accuracy_score(y_test, y_pred_tree))
+
+
+
+
+
from matplotlib.colors import ListedColormap
+
+def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):
+ x1s = np.linspace(axes[0], axes[1], 100)
+ x2s = np.linspace(axes[2], axes[3], 100)
+ x1, x2 = np.meshgrid(x1s, x2s)
+ X_new = np.c_[x1.ravel(), x2.ravel()]
+ y_pred = clf.predict(X_new).reshape(x1.shape)
+ custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
+ plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
+ if contour:
+ custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
+ plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
+ plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha)
+ plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha)
+ plt.axis(axes)
+ plt.xlabel(r"$x_1$", fontsize=18)
+ plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
+plt.figure(figsize=(11,4))
+plt.subplot(121)
+plot_decision_boundary(tree_clf, X, y)
+plt.title("Decision Tree", fontsize=14)
+plt.subplot(122)
+plot_decision_boundary(bag_clf, X, y)
+plt.title("Decision Trees with Bagging", fontsize=14)
+plt.show()
+
@@ -215,6 +291,14 @@ Example will be added here.
40
41
42
+ 43
+ 44
+ 45
+ 46
+ 47
+ 48
+ ...
+ 50
»
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-bs.html b/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
index 71dfb146f..ec2e98fd3 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
@@ -64,40 +64,55 @@ Automatically generated HTML file from DocOnce source
('Classification tree, how to split nodes', 2, None, '___sec13'),
('Visualizing the Tree, Classification', 2, None, '___sec14'),
('Visualizing the Tree, The Moons', 2, None, '___sec15'),
- ('Computing the Gini index', 2, None, '___sec16'),
+ ('Algorithms for Setting up Decision Trees', 2, None, '___sec16'),
+ ('The CART algorithm for Classification', 2, None, '___sec17'),
+ ('The CART algorithm for Regression', 2, None, '___sec18'),
+ ('Computing the Gini index', 2, None, '___sec19'),
('Simple Python Code to read in Data and perform Classification',
2,
None,
- '___sec17'),
- ('Computing the Gini Factor', 2, None, '___sec18'),
- ('Entropy and the ID3 algorithm', 2, None, '___sec19'),
- ('Implementing the ID3 Algorithm', 2, None, '___sec20'),
+ '___sec20'),
+ ('Computing the Gini Factor', 2, None, '___sec21'),
+ ('Entropy and the ID3 algorithm', 2, None, '___sec22'),
+ ('Implementing the ID3 Algorithm', 2, None, '___sec23'),
('Cancer Data again now with Decision Trees and other Methods',
2,
None,
- '___sec21'),
- ('Another example, the moons again', 2, None, '___sec22'),
- ('Playing around with regions', 2, None, '___sec23'),
- ('Regression trees', 2, None, '___sec24'),
- ('Final regressor code', 2, None, '___sec25'),
- ('Pros and cons of trees, pros', 2, None, '___sec26'),
- ('Disadvantages', 2, None, '___sec27'),
- ('Bagging', 2, None, '___sec28'),
- ('More bagging', 2, None, '___sec29'),
- ('Simple Voting Example, head or tail', 2, None, '___sec30'),
- ('Using the Voting Classifier', 2, None, '___sec31'),
+ '___sec24'),
+ ('Another example, the moons again', 2, None, '___sec25'),
+ ('Playing around with regions', 2, None, '___sec26'),
+ ('Regression trees', 2, None, '___sec27'),
+ ('Final regressor code', 2, None, '___sec28'),
+ ('Pros and cons of trees, pros', 2, None, '___sec29'),
+ ('Disadvantages', 2, None, '___sec30'),
+ ('From a Single Tree to Many Trees, that is meet the Jungle of '
+ 'Methods',
+ 2,
+ None,
+ '___sec31'),
+ ('Bagging', 2, None, '___sec32'),
+ ('More bagging', 2, None, '___sec33'),
+ ('Simple Voting Example, head or tail', 2, None, '___sec34'),
+ ('Using the Voting Classifier', 2, None, '___sec35'),
('Please, not the moons again! Voting and Bagging',
2,
None,
- '___sec32'),
- ('Now Bagging', 2, None, '___sec33'),
- ('Random forests', 2, None, '___sec34'),
- ('A simple scikit-learn example', 2, None, '___sec35'),
- ('Then random forests', 2, None, '___sec36'),
- ('Feature Importance', 2, None, '___sec37'),
- ('Boosting: AdaBoost', 2, None, '___sec38'),
- ('Gradient Boosting', 2, None, '___sec39'),
- ('Gradient Boots with Early Stopping', 2, None, '___sec40')]}
+ '___sec36'),
+ ('Now Bagging', 2, None, '___sec37'),
+ ('Random forests', 2, None, '___sec38'),
+ ('A simple scikit-learn example', 2, None, '___sec39'),
+ ('Then random forests', 2, None, '___sec40'),
+ ('Feature Importance', 2, None, '___sec41'),
+ ("Boosting, a Bird'e Eye", 2, None, '___sec42'),
+ ('Adaptive boosting: AdaBoost, Basic Algorithm',
+ 2,
+ None,
+ '___sec43'),
+ ('AdaBoost Examples', 2, None, '___sec44'),
+ ('Gradient boosting: Basic Algorithm', 2, None, '___sec45'),
+ ('Gradient Boosting, Examples', 2, None, '___sec46'),
+ ('Gradient Boots with Early Stopping', 2, None, '___sec47'),
+ ('XGBoost: Extreme Gradient Boosting', 2, None, '___sec48')]}
end of tocinfo -->
@@ -151,31 +166,39 @@ MathJax.Hub.Config({
Classification tree, how to split nodes
Visualizing the Tree, Classification
Visualizing the Tree, The Moons
- Computing the Gini index
- Simple Python Code to read in Data and perform Classification
- Computing the Gini Factor
- Entropy and the ID3 algorithm
- Implementing the ID3 Algorithm
- Cancer Data again now with Decision Trees and other Methods
- Another example, the moons again
- Playing around with regions
- Regression trees
- Final regressor code
- Pros and cons of trees, pros
- Disadvantages
- Bagging
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier
- Please, not the moons again! Voting and Bagging
- Now Bagging
- Random forests
- A simple scikit-learn example
- Then random forests
- Feature Importance
- Boosting: AdaBoost
- Gradient Boosting
- Gradient Boots with Early Stopping
+ Algorithms for Setting up Decision Trees
+ The CART algorithm for Classification
+ The CART algorithm for Regression
+ Computing the Gini index
+ Simple Python Code to read in Data and perform Classification
+ Computing the Gini Factor
+ Entropy and the ID3 algorithm
+ Implementing the ID3 Algorithm
+ Cancer Data again now with Decision Trees and other Methods
+ Another example, the moons again
+ Playing around with regions
+ Regression trees
+ Final regressor code
+ Pros and cons of trees, pros
+ Disadvantages
+ From a Single Tree to Many Trees, that is meet the Jungle of Methods
+ Bagging
+ More bagging
+ Simple Voting Example, head or tail
+ Using the Voting Classifier
+ Please, not the moons again! Voting and Bagging
+ Now Bagging
+ Random forests
+ A simple scikit-learn example
+ Then random forests
+ Feature Importance
+ Boosting, a Bird'e Eye
+ Adaptive boosting: AdaBoost, Basic Algorithm
+ AdaBoost Examples
+ Gradient boosting: Basic Algorithm
+ Gradient Boosting, Examples
+ Gradient Boots with Early Stopping
+ XGBoost: Extreme Gradient Boosting
@@ -210,7 +233,7 @@ MathJax.Hub.Config({
[2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
-
Nov 1, 2019
Nov 2, 2019
@@ -234,7 +257,7 @@ MathJax.Hub.Config({
9
10
...
- 42
+ 50
»
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html b/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
index 758d6151d..5b1289cc6 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
@@ -148,7 +148,7 @@ MathJax.Hub.Config({
[2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
-
Nov 1, 2019
Nov 2, 2019
@@ -194,6 +194,9 @@ given some assumptions, make predictions about the target feature value
A typical Decision Tree with its pertinent Jargon, Classification Problem
+
+
+Figure to come here.
@@ -691,7 +694,31 @@ os.system(cmd)
-Computing the Gini index
+Algorithms for Setting up Decision Trees
+Two algorithms stand out in the set up of decision trees:
+
+
+- The CART (Classification And Regression Tree) algorithm for both classification and regression
+- The ID3 algorithm based on the computation of the information gain for classification
+
+
+
+We discuss both algorithms with applications here. The popular library -Scikit-Learn_ uses the CART algorithm. For classification problems you can use either the gini index or the entropy to split a tree in two branches.
+
+
+
+
+The CART algorithm for Classification
+
+
+
+
+The CART algorithm for Regression
+
+
+
+
+Computing the Gini index
The example we will look at is a classical one in many Machine
@@ -732,7 +759,7 @@ The table here summarizes the various attributes and
-Simple Python Code to read in Data and perform Classification
+Simple Python Code to read in Data and perform Classification
@@ -809,7 +836,7 @@ os.system(cmd)
-Computing the Gini Factor
+Computing the Gini Factor
The above functions (gini, entropy and misclassification error) are
@@ -888,7 +915,7 @@ split = get_split(dataset)
-Entropy and the ID3 algorithm
+Entropy and the ID3 algorithm
ID3, learns decision trees by constructing
@@ -925,7 +952,7 @@ attributes at each step while growing the tree.
-Implementing the ID3 Algorithm
+Implementing the ID3 Algorithm
@@ -1122,7 +1149,7 @@ attributes at each step while growing the tree.
-Cancer Data again now with Decision Trees and other Methods
+Cancer Data again now with Decision Trees and other Methods
@@ -1172,7 +1199,7 @@ deep_tree_clf.fit(X_train_scaled, y_train)
-Another example, the moons again
+Another example, the moons again
@@ -1245,7 +1272,7 @@ plt.show()
-Playing around with regions
+Playing around with regions
@@ -1274,7 +1301,7 @@ plt.show()
-Regression trees
+Regression trees
@@ -1297,7 +1324,7 @@ tree_reg.fit(X, y)
-Final regressor code
+Final regressor code
@@ -1376,7 +1403,7 @@ plt.show()
-Pros and cons of trees, pros
+Pros and cons of trees, pros
- White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines)
@@ -1391,7 +1418,7 @@ plt.show()
-Disadvantages
+Disadvantages
- Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches
@@ -1409,7 +1436,32 @@ However, by aggregating many decision trees, using methods like bagging, random
-Bagging
+From a Single Tree to Many Trees, that is meet the Jungle of Methods
+
+
+As stated above and seen in many of the examples discussed here about
+a single decision tree, we often end up overfitting our training
+data. This normally means that we have a high variance. Can we reduce
+the variance of a statistical learning method?
+
+
+This leads us to a set of different methods that can combine different
+machine learning algorithms or just use one of them to construct forests and jungles of trees, homogeneous ones or heterogenous ones. These methods are recognized by different names which we will try to explain here. These are
+
+
+- Votign classifiers
+- Bagging and Pasting
+- Random forests
+- Boosting methods
+
+
+
+We discuss these methods here.
+
+
+
+
+Bagging
The plain decision trees suffer from high
@@ -1428,7 +1480,7 @@ learning method.
-More bagging
+More bagging
Bagging typically results in improved accuracy
@@ -1457,7 +1509,7 @@ predictor, averaged over all \( B \) trees.
-Simple Voting Example, head or tail
+Simple Voting Example, head or tail
@@ -1478,7 +1530,7 @@ plt.show()
-Using the Voting Classifier
+Using the Voting Classifier
@@ -1530,7 +1582,7 @@ voting_clf.fit(X_train, y_train)
-Please, not the moons again! Voting and Bagging
+Please, not the moons again! Voting and Bagging
@@ -1589,7 +1641,7 @@ voting_clf.fit(X_train, y_train)
-Now Bagging
+Now Bagging
@@ -1651,7 +1703,7 @@ plt.show()
-Random forests
+Random forests
Random forests provide an improvement over bagged trees by way of a
@@ -1697,7 +1749,7 @@ this setting.
-A simple scikit-learn example
+A simple scikit-learn example
@@ -1716,7 +1768,7 @@ accuracy = cross_validate(Random_Forest_model,X,Y,cv=Then random forests
+Then random forests
@@ -1739,7 +1791,7 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
-Feature Importance
+Feature Importance
Example will be added here.
@@ -1747,8 +1799,17 @@ Example will be added here.
-Boosting: AdaBoost
+Boosting, a Bird'e Eye
+
+
+
+Adaptive boosting: AdaBoost, Basic Algorithm
+
+
+
+
+AdaBoost Examples
@@ -1788,7 +1849,12 @@ plt.show()
-Gradient Boosting
+Gradient boosting: Basic Algorithm
+
+
+
+
+Gradient Boosting, Examples
@@ -1879,7 +1945,7 @@ plt.show()
-Gradient Boots with Early Stopping
+Gradient Boots with Early Stopping
@@ -1943,6 +2009,11 @@ error_going_up = 0
+
+XGBoost: Extreme Gradient Boosting
+
+
+
-
@@ -200,6 +215,9 @@ given some assumptions, make predictions about the target feature value
+Figure to come here. +
@@ -675,7 +693,30 @@ os.system(cmd)
-
+
+
+
+
+
+
+
+
+
The example we will look at is a classical one in many Machine @@ -715,7 +756,7 @@ The table here summarizes the various attributes and
-
@@ -791,7 +832,7 @@ os.system(cmd)
-
The above functions (gini, entropy and misclassification error) are @@ -869,7 +910,7 @@ split = get_split(dataset)
-
ID3, learns decision trees by constructing @@ -905,7 +946,7 @@ attributes at each step while growing the tree.
-
@@ -1101,7 +1142,7 @@ attributes at each step while growing the tree.
-
@@ -1150,7 +1191,7 @@ deep_tree_clf.fit(X_train_scaled, y_train)
-
@@ -1222,7 +1263,7 @@ plt.show()
-
@@ -1250,7 +1291,7 @@ plt.show()
-
@@ -1272,7 +1313,7 @@ tree_reg.fit(X, y)
-
@@ -1350,7 +1391,7 @@ plt.show()
-
-
+As stated above and seen in many of the examples discussed here about +a single decision tree, we often end up overfitting our training +data. This normally means that we have a high variance. Can we reduce +the variance of a statistical learning method? + +
+This leads us to a set of different methods that can combine different +machine learning algorithms or just use one of them to construct forests and jungles of trees, homogeneous ones or heterogenous ones. These methods are recognized by different names which we will try to explain here. These are + +
+
+
+
The plain decision trees suffer from high @@ -1400,7 +1465,7 @@ learning method.
-
Bagging typically results in improved accuracy @@ -1429,7 +1494,7 @@ predictor, averaged over all \( B \) trees.
-
@@ -1449,7 +1514,7 @@ plt.show()
-
@@ -1500,7 +1565,7 @@ voting_clf.fit(X_train, y_train)
-
@@ -1558,7 +1623,7 @@ voting_clf.fit(X_train, y_train)
-
@@ -1619,7 +1684,7 @@ plt.show()
-
Random forests provide an improvement over bagged trees by way of a @@ -1663,7 +1728,7 @@ this setting.
-
@@ -1681,7 +1746,7 @@ accuracy = cross_validate(Random_Forest_model,X,Y,cv=Then random forests
+
@@ -1703,7 +1768,7 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
Example will be added here.
@@ -1711,8 +1776,17 @@ Example will be added here.
+
+
@@ -1751,7 +1825,12 @@ plt.show()
+
@@ -1841,7 +1920,7 @@ plt.show()
@@ -1903,6 +1982,9 @@ error_going_up = 0
print("Minimum validation MSE:", min_val_error)
Then random forests
-Feature Importance
+Feature Importance
-Boosting: AdaBoost
+Boosting, a Bird'e Eye
+
+
+Adaptive boosting: AdaBoost, Basic Algorithm
+
+
+
+AdaBoost Examples
-Gradient Boosting
+Gradient boosting: Basic Algorithm
+
+
+
+Gradient Boosting, Examples
-Gradient Boots with Early Stopping
+Gradient Boots with Early Stopping
+
+
+
-
@@ -205,6 +220,9 @@ given some assumptions, make predictions about the target feature value
+Figure to come here. +
@@ -680,7 +698,30 @@ os.system(cmd)
-
+
+
+
+
+
+
+
+
+
The example we will look at is a classical one in many Machine @@ -720,7 +761,7 @@ The table here summarizes the various attributes and
-
@@ -796,7 +837,7 @@ os.system(cmd)
-
The above functions (gini, entropy and misclassification error) are @@ -874,7 +915,7 @@ split = get_split(dataset)
-
ID3, learns decision trees by constructing @@ -910,7 +951,7 @@ attributes at each step while growing the tree.
-
@@ -1106,7 +1147,7 @@ attributes at each step while growing the tree.
-
@@ -1155,7 +1196,7 @@ deep_tree_clf.fit(X_train_scaled, y_train)
-
@@ -1227,7 +1268,7 @@ plt.show()
-
@@ -1255,7 +1296,7 @@ plt.show()
-
@@ -1277,7 +1318,7 @@ tree_reg.fit(X, y)
-
@@ -1355,7 +1396,7 @@ plt.show()
-
-
+As stated above and seen in many of the examples discussed here about +a single decision tree, we often end up overfitting our training +data. This normally means that we have a high variance. Can we reduce +the variance of a statistical learning method? + +
+This leads us to a set of different methods that can combine different +machine learning algorithms or just use one of them to construct forests and jungles of trees, homogeneous ones or heterogenous ones. These methods are recognized by different names which we will try to explain here. These are + +
+
+
+
The plain decision trees suffer from high @@ -1405,7 +1470,7 @@ learning method.
-
Bagging typically results in improved accuracy @@ -1434,7 +1499,7 @@ predictor, averaged over all \( B \) trees.
-
@@ -1454,7 +1519,7 @@ plt.show()
-
@@ -1505,7 +1570,7 @@ voting_clf.fit(X_train, y_train)
-
@@ -1563,7 +1628,7 @@ voting_clf.fit(X_train, y_train)
-
@@ -1624,7 +1689,7 @@ plt.show()
-
Random forests provide an improvement over bagged trees by way of a @@ -1668,7 +1733,7 @@ this setting.
-
@@ -1686,7 +1751,7 @@ accuracy = cross_validate(Random_Forest_mode
-
@@ -1708,7 +1773,7 @@ np.sum(y_pred =
Example will be added here.
@@ -1716,8 +1781,17 @@ Example will be added here.
+
+
@@ -1756,7 +1830,12 @@ plt.show()
+
@@ -1846,7 +1925,7 @@ plt.show()
@@ -1908,6 +1987,9 @@ error_going_up = print("Minimum validation MSE:", min_val_error)
-Feature Importance
+Feature Importance
-Boosting: AdaBoost
+Boosting, a Bird'e Eye
+
+
+Adaptive boosting: AdaBoost, Basic Algorithm
+
+
+
+AdaBoost Examples
-Gradient Boosting
+Gradient boosting: Basic Algorithm
+
+
+
+Gradient Boosting, Examples
-Gradient Boots with Early Stopping
+Gradient Boots with Early Stopping
+
+
+