-In the figure here we present a decision tree obtained from a classification problem -
@@ -206,7 +197,7 @@ In the figure here we present a decision tree obtained from a classification pro
-In the figure we present a decision tree obtained from a simple regression problem +The overarching approach to decision trees is a top-down approach. + +
@@ -207,7 +211,7 @@ In the figure we present a decision tree obtained from a simple regression prob
-The overarching approach to decision trees is a top-down approach. +In simplified terms, the process of training a decision tree and +predicting the target features of query instances is as follows: -
@@ -218,7 +212,7 @@ node.
-In simplified terms, the process of training a decision tree and -predicting the target features of query instances is as follows: -
import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.linear_model import LinearRegression
-Then we are essentially done!
+steps=250
+distance=0
+x=0
+distance_list=[]
+steps_list=[]
+while x<steps:
+ distance+=np.random.randint(-1,2)
+ distance_list.append(distance)
+ x+=1
+ steps_list.append(x)
+plt.plot(steps_list,distance_list, color='green', label="Random Walk Data")
+
+steps_list=np.asarray(steps_list)
+distance_list=np.asarray(distance_list)
+
+X=steps_list[:,np.newaxis]
+
+#Polynomial fits
+
+#Degree 2
+poly_features=PolynomialFeatures(degree=2, include_bias=False)
+X_poly=poly_features.fit_transform(X)
+
+lin_reg=LinearRegression()
+poly_fit=lin_reg.fit(X_poly,distance_list)
+b=lin_reg.coef_
+c=lin_reg.intercept_
+print ("2nd degree coefficients:")
+print ("zero power: ",c)
+print ("first power: ", b[0])
+print ("second power: ",b[1])
+
+z = np.arange(0, steps, .01)
+z_mod=b[1]*z**2+b[0]*z+c
+
+fit_mod=b[1]*X**2+b[0]*X+c
+plt.plot(z, z_mod, color='r', label="2nd Degree Fit")
+plt.title("Polynomial Regression")
+
+plt.xlabel("Steps")
+plt.ylabel("Distance")
+
+#Degree 10
+poly_features10=PolynomialFeatures(degree=10, include_bias=False)
+X_poly10=poly_features10.fit_transform(X)
+
+poly_fit10=lin_reg.fit(X_poly10,distance_list)
+
+y_plot=poly_fit10.predict(X_poly10)
+plt.plot(X, y_plot, color='black', label="10th Degree Fit")
+
+plt.legend()
+plt.show()
+
+
+#Decision Tree Regression
+from sklearn.tree import DecisionTreeRegressor
+regr_1=DecisionTreeRegressor(max_depth=2)
+regr_2=DecisionTreeRegressor(max_depth=5)
+regr_3=DecisionTreeRegressor(max_depth=7)
+regr_1.fit(X, distance_list)
+regr_2.fit(X, distance_list)
+regr_3.fit(X, distance_list)
+
+X_test = np.arange(0.0, steps, 0.01)[:, np.newaxis]
+y_1 = regr_1.predict(X_test)
+y_2 = regr_2.predict(X_test)
+y_3=regr_3.predict(X_test)
+
+# Plot the results
+plt.figure()
+plt.scatter(X, distance_list, s=2.5, c="black", label="data")
+plt.plot(X_test, y_1, color="red",
+ label="max_depth=2", linewidth=2)
+plt.plot(X_test, y_2, color="green", label="max_depth=5", linewidth=2)
+plt.plot(X_test, y_3, color="m", label="max_depth=7", linewidth=2)
+
+plt.xlabel("Data")
+plt.ylabel("Darget")
+plt.title("Decision Tree Regression")
+plt.legend()
+plt.show()
+
@@ -219,7 +291,7 @@ Then we are essentially done!
+There are mainly two steps - -
import numpy as np
-import matplotlib.pyplot as plt
-from sklearn.preprocessing import PolynomialFeatures
-from sklearn.linear_model import LinearRegression
+
+- We split the predictor space (the set of possible values \( x_1,x_2,\dots, x_p \)) into \( J \) distinct and non-non-overlapping regions, \( R_1,R_2,\dots,R_J \).
+- For every observation that falls into the region \( R_j \) , we make the same prediction, which is simply the mean of the response values for the training observations in \( R_j \).
+
-steps=250
+How do we construct the regions \( R_1,\dots,R_J \)? In theory, the
+regions could have any shape. However, we choose to divide the
+predictor space into high-dimensional rectangles, or boxes, for
+simplicity and for ease of interpretation of the resulting predictive
+model. The goal is to find boxes \( R_1,\dots,R_J \) that minimize the
+MSE, given by
-distance=0
-x=0
-distance_list=[]
-steps_list=[]
-while x<steps:
- distance+=np.random.randint(-1,2)
- distance_list.append(distance)
- x+=1
- steps_list.append(x)
-plt.plot(steps_list,distance_list, color='green', label="Random Walk Data")
+$$
+\sum_{j=1}^J\sum_{i\in R_j}(y_i-\overline{y}_{R_j})^2,
+$$
-steps_list=np.asarray(steps_list)
-distance_list=np.asarray(distance_list)
+
+where \( \overline{y}_{R_j} \) is the mean response for the training observations
+within box \( j \).
-X=steps_list[:,np.newaxis]
-
-#Polynomial fits
-
-#Degree 2
-poly_features=PolynomialFeatures(degree=2, include_bias=False)
-X_poly=poly_features.fit_transform(X)
-
-lin_reg=LinearRegression()
-poly_fit=lin_reg.fit(X_poly,distance_list)
-b=lin_reg.coef_
-c=lin_reg.intercept_
-print ("2nd degree coefficients:")
-print ("zero power: ",c)
-print ("first power: ", b[0])
-print ("second power: ",b[1])
-
-z = np.arange(0, steps, .01)
-z_mod=b[1]*z**2+b[0]*z+c
-
-fit_mod=b[1]*X**2+b[0]*X+c
-plt.plot(z, z_mod, color='r', label="2nd Degree Fit")
-plt.title("Polynomial Regression")
-
-plt.xlabel("Steps")
-plt.ylabel("Distance")
-
-#Degree 10
-poly_features10=PolynomialFeatures(degree=10, include_bias=False)
-X_poly10=poly_features10.fit_transform(X)
-
-poly_fit10=lin_reg.fit(X_poly10,distance_list)
-
-y_plot=poly_fit10.predict(X_poly10)
-plt.plot(X, y_plot, color='black', label="10th Degree Fit")
-
-plt.legend()
-plt.show()
-
-
-#Decision Tree Regression
-from sklearn.tree import DecisionTreeRegressor
-regr_1=DecisionTreeRegressor(max_depth=2)
-regr_2=DecisionTreeRegressor(max_depth=5)
-regr_3=DecisionTreeRegressor(max_depth=7)
-regr_1.fit(X, distance_list)
-regr_2.fit(X, distance_list)
-regr_3.fit(X, distance_list)
-
-X_test = np.arange(0.0, steps, 0.01)[:, np.newaxis]
-y_1 = regr_1.predict(X_test)
-y_2 = regr_2.predict(X_test)
-y_3=regr_3.predict(X_test)
-
-# Plot the results
-plt.figure()
-plt.scatter(X, distance_list, s=2.5, c="black", label="data")
-plt.plot(X_test, y_1, color="red",
- label="max_depth=2", linewidth=2)
-plt.plot(X_test, y_2, color="green", label="max_depth=5", linewidth=2)
-plt.plot(X_test, y_3, color="m", label="max_depth=7", linewidth=2)
-
-plt.xlabel("Data")
-plt.ylabel("Darget")
-plt.title("Decision Tree Regression")
-plt.legend()
-plt.show()
-
@@ -298,7 +224,7 @@ plt.show()
-There are mainly two steps - -
-where \( \overline{y}_{R_j} \) is the mean response for the training observations -within box \( j \). +The approach is top-down because it begins at the top of the tree (all +observations belong to a single region) and then successively splits +the predictor space; each split is indicated via two new branches +further down on the tree. It is greedy because at each step of the +tree-building process, the best split is made at that particular step, +rather than looking ahead and picking a split that will lead to a +better tree in some future step.
@@ -231,7 +216,7 @@ within box \( j \).
-Unfortunately, it is computationally infeasible to consider every -possible partition of the feature space into \( J \) boxes. The common -strategy is to take a top-down approach +In order to implement the recursive binary splitting we start by selecting +the predictor \( x_j \) and a cutpoint \( s \) that splits the predictor space into two regions \( R_1 \) and \( R_2 \) +$$ +\left\{X\vert x_j < s\right\}, +$$ + +and +$$ +\left\{X\vert x_j \geq s\right\}, +$$ + +so that we obtain the lowest MSE, that is +$$ +\sum_{i:x_i\in R_j}(y_i-\overline{y}_{R_1})^2+\sum_{i:x_i\in R_2}(y_i-\overline{y}_{R_2})^2, +$$
-The approach is top-down because it begins at the top of the tree (all -observations belong to a single region) and then successively splits -the predictor space; each split is indicated via two new branches -further down on the tree. It is greedy because at each step of the -tree-building process, the best split is made at that particular step, -rather than looking ahead and picking a split that will lead to a -better tree in some future step. +which we want to minimize by considering all predictors +\( x_1,x_2,\dots,x_p \). We consider also all possible values of \( s \) for +each predictor. These values could be determined by randomly assigned +numbers or by starting at the midpoint and then proceed till we find +an optimal value. + +
+For any \( j \) and \( s \), we define the pair of half-planes where +\( \overline{y}_{R_1} \) is the mean response for the training +observations in \( R_1(j,s) \), and \( \overline{y}_{R_2} \) is the mean +response for the training observations in \( R_2(j,s) \). + +
+Finding the values of \( j \) and \( s \) that minimize the above equation can be +done quite quickly, especially when the number of features \( p \) is not +too large. + +
+Next, we repeat the process, looking +for the best predictor and best cutpoint in order to split the data +further so as to minimize the MSE within each of the resulting +regions. However, this time, instead of splitting the entire predictor +space, we split one of the two previously identified regions. We now +have three regions. Again, we look to split one of these three regions +further, so as to minimize the MSE. The process continues until a +stopping criterion is reached; for instance, we may continue until no +region contains more than five observations.
@@ -223,7 +249,7 @@ better tree in some future step.
- + -
-In order to implement the recursive binary splitting we start by selecting -the predictor \( x_j \) and a cutpoint \( s \) that splits the predictor space into two regions \( R_1 \) and \( R_2 \) -$$ -\left\{X\vert x_j < s\right\}, -$$ - -and -$$ -\left\{X\vert x_j \geq s\right\}, -$$ - -so that we obtain the lowest MSE, that is -$$ -\sum_{i:x_i\in R_j}(y_i-\overline{y}_{R_1})^2+\sum_{i:x_i\in R_2}(y_i-\overline{y}_{R_2})^2, -$$ +The above procedure is rather straightforward, but leads often to +overfitting and unnecessarily large and complicated trees. The basic +idea is to grow a large tree \( T_0 \) and then prune it back in order to +obtain a subtree. A smaller tree with fewer splits (fewer regions) can +lead to smaller variance and better interpretation at the cost of a +little more bias.
-which we want to minimize by considering all predictors -\( x_1,x_2,\dots,x_p \). We consider also all possible values of \( s \) for -each predictor. These values could be determined by randomly assigned -numbers or by starting at the midpoint and then proceed till we find -an optimal value. - -
-For any \( j \) and \( s \), we define the pair of half-planes where -\( \overline{y}_{R_1} \) is the mean response for the training -observations in \( R_1(j,s) \), and \( \overline{y}_{R_2} \) is the mean -response for the training observations in \( R_2(j,s) \). - -
-Finding the values of \( j \) and \( s \) that minimize the above equation can be -done quite quickly, especially when the number of features \( p \) is not -too large. - -
-Next, we repeat the process, looking -for the best predictor and best cutpoint in order to split the data -further so as to minimize the MSE within each of the resulting -regions. However, this time, instead of splitting the entire predictor -space, we split one of the two previously identified regions. We now -have three regions. Again, we look to split one of these three regions -further, so as to minimize the MSE. The process continues until a -stopping criterion is reached; for instance, we may continue until no -region contains more than five observations. +The so-called Cost complexity pruning algorithm gives us a +way to do just this. Rather than considering every possible subtree, +we consider a sequence of trees indexed by a nonnegative tuning +parameter \( \alpha \).
@@ -256,7 +218,7 @@ region contains more than five observations.
- + -
-The above procedure is rather straightforward, but leads often to -overfitting and unnecessarily large and complicated trees. The basic -idea is to grow a large tree \( T_0 \) and then prune it back in order to -obtain a subtree. A smaller tree with fewer splits (fewer regions) can -lead to smaller variance and better interpretation at the cost of a -little more bias. +The tuning parameter \( \alpha \) controls a trade-off between the subtree’s +com- plexity and its fit to the training data. When \( \alpha = 0 \), then the +subtree \( T \) will simply equal \( T_0 \), +because then the above equation just measures the +training error. +However, as \( \alpha \) increases, there is a price to pay for +having a tree with many terminal nodes. The above equation will +tend to be minimized for a smaller subtree.
-The so-called Cost complexity pruning algorithm gives us a -way to do just this. Rather than considering every possible subtree, -we consider a sequence of trees indexed by a nonnegative tuning -parameter \( \alpha \). +It turns out that as we increase \( \alpha \) from zero +branches get pruned from the tree in a nested and predictable fashion, +so obtaining the whole sequence of subtrees as a function of \( \alpha \) is +easy. We can select a value of \( \alpha \) using a validation set or using +cross-validation. We then return to the full data set and obtain the +subtree corresponding to \( \alpha \).
@@ -225,7 +231,7 @@ parameter \( \alpha \).
-The tuning parameter \( \alpha \) controls a trade-off between the subtree’s -com- plexity and its fit to the training data. When \( \alpha = 0 \), then the -subtree \( T \) will simply equal \( T_0 \), -because then the above equation just measures the -training error. -However, as \( \alpha \) increases, there is a price to pay for -having a tree with many terminal nodes. The above equation will -tend to be minimized for a smaller subtree. +
+ +
-It turns out that as we increase \( \alpha \) from zero -branches get pruned from the tree in a nested and predictable fashion, -so obtaining the whole sequence of subtrees as a function of \( \alpha \) is -easy. We can select a value of \( \alpha \) using a validation set or using -cross-validation. We then return to the full data set and obtain the -subtree corresponding to \( \alpha \).
@@ -237,7 +227,7 @@ subtree corresponding to \( \alpha \).
-
- -
@@ -233,7 +219,7 @@ MathJax.Hub.Config({
-A classification tree is very similar to a regression tree, except -that it is used to predict a qualitative response rather than a -quantitative one. Recall that for a regression tree, the predicted -response for an observation is given by the mean response of the -training observations that belong to the same terminal node. In -contrast, for a classification tree, we predict that each observation -belongs to the most commonly occurring class of training observations -in the region to which it belongs. In interpreting the results of a -classification tree, we are often interested not only in the class -prediction corresponding to a particular terminal node region, but -also in the class proportions among the training observations that -fall into that region. +The task of growing a +classification tree is quite similar to the task of growing a +regression tree. Just as in the regression setting, we use recursive +binary splitting to grow a classification tree. However, in the +classification setting, the MSE cannot be used as a criterion for making +the binary splits. A natural alternative to MSE is the classification +error rate. Since we plan to assign an observation in a given region +to the most commonly occurring error rate class of training +observations in that region, the classification error rate is simply +the fraction of the training observations in that region that do not +belong to the most common class. + +
+When building a classification tree, either the Gini index or the +entropy are typically used to evaluate the quality of a particular +split, since these two approaches are more sensitive to node purity +than is the classification error rate.
@@ -225,7 +224,7 @@ fall into that region.
-The task of growing a -classification tree is quite similar to the task of growing a -regression tree. Just as in the regression setting, we use recursive -binary splitting to grow a classification tree. However, in the -classification setting, the MSE cannot be used as a criterion for making -the binary splits. A natural alternative to MSE is the classification -error rate. Since we plan to assign an observation in a given region -to the most commonly occurring error rate class of training -observations in that region, the classification error rate is simply -the fraction of the training observations in that region that do not -belong to the most common class. +If our targets are the outcome of a classification process that takes +for example \( k=1,2,\dots,K \) values, the only thing we need to think of +is to set up the splitting criteria for each node.
-When building a classification tree, either the Gini index or the -entropy are typically used to evaluate the quality of a particular -split, since these two approaches are more sensitive to node purity -than is the classification error rate. +We define a PDF \( p_{mk} \) that represents the number of observations of +a class \( k \) in a region \( R_m \) with \( N_m \) observations. We represent +this likelihood function in terms of the proportion \( I(y_i=k) \) of +observations of this class in the region \( R_m \) as + +$$ +p_{mk} = \frac{1}{N_m}\sum_{x_i\in R_m}I(y_i=k). +$$ + +
+We let \( p_{mk} \) represent the majority class of observations in region +\( m \). The three most common ways of splitting a node are given by + +
@@ -230,7 +250,7 @@ than is the classification error rate.
-If our targets are the outcome of a classification process that takes -for example \( k=1,2,\dots,K \) values, the only thing we need to think of -is to set up the splitting criteria for each node. -
-We define a PDF \( p_{mk} \) that represents the number of observations of -a class \( k \) in a region \( R_m \) with \( N_m \) observations. We represent -this likelihood function in terms of the proportion \( I(y_i=k) \) of -observations of this class in the region \( R_m \) as + +
import os
+from sklearn.datasets import load_breast_cancer
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import confusion_matrix
+from sklearn.tree import export_graphviz
-$$
-p_{mk} = \frac{1}{N_m}\sum_{x_i\in R_m}I(y_i=k).
-$$
-
-
-We let \( p_{mk} \) represent the majority class of observations in region
-\( m \). The three most common ways of splitting a node are given by
-
-
@@ -256,7 +241,7 @@ $$
-
import os
-from sklearn.datasets import load_breast_cancer
+# Common imports
+import numpy as np
+from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import confusion_matrix
+from sklearn.datasets import make_moons
from sklearn.tree import export_graphviz
-
-from IPython.display import Image
from pydot import graph_from_dot_data
import pandas as pd
-import numpy as np
+import os
-
-cancer = load_breast_cancer()
-X = pd.DataFrame(cancer.data, columns=cancer.feature_names)
-print(X)
-y = pd.Categorical.from_codes(cancer.target, cancer.target_names)
-y = pd.get_dummies(y)
-print(y)
-X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)
+np.random.seed(42)
+X, y = make_moons(n_samples=100, noise=0.25, random_state=53)
+X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)
tree_clf = DecisionTreeClassifier(max_depth=5)
tree_clf.fit(X_train, y_train)
export_graphviz(
tree_clf,
- out_file="DataFiles/cancer.dot",
- feature_names=cancer.feature_names,
- class_names=cancer.target_names,
+ out_file="DataFiles/moons.dot",
rounded=True,
filled=True
)
-cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
+cmd = 'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png'
os.system(cmd)
@@ -247,7 +232,7 @@ os.system(cmd)
+The example we will look at is a classical one in many Machine +Learning applications. Based on various meteorological features, we +have several so-called attributes which decide whether we at the end +will do some outdoor activity like skiing, going for a bike ride etc +etc. The table here contains the feautures outlook, temperature, +humidity and wind. The target or output is whether we ride +(True=1) or whether we do something else that day (False=0). The +attributes for each feature are then sunny, overcast and rain for the +outlook, hot, cold and mild for temperature, high and normal for +humidity and weak and strong for wind. - -
# Common imports
-import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.datasets import make_moons
-from sklearn.tree import export_graphviz
-from pydot import graph_from_dot_data
-import pandas as pd
-import os
+
+The table here summarizes the various attributes and
-np.random.seed(42)
-X, y = make_moons(n_samples=100, noise=0.25, random_state=53)
-X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)
-tree_clf = DecisionTreeClassifier(max_depth=5)
-tree_clf.fit(X_train, y_train)
-
-export_graphviz(
- tree_clf,
- out_file="DataFiles/moons.dot",
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png'
-os.system(cmd)
-
| Day | Outlook | Temperature | Humidity | Wind | Ride |
| 1 | Sunny | Hot | High | Weak | 0 |
| 2 | Sunny | Hot | High | Strong | 1 |
| 3 | Overcast | Hot | High | Weak | 1 |
| 4 | Rain | Mild | High | Weak | 1 |
| 5 | Rain | Cool | Normal | Weak | 1 |
| 6 | Rain | Cool | Normal | Strong | 0 |
| 7 | Overcast | Cool | Normal | Strong | 1 |
| 8 | Sunny | Mild | High | Weak | 0 |
| 9 | Sunny | Cool | Normal | Weak | 1 |
| 10 | Rain | Mild | Normal | Weak | 1 |
| 11 | Sunny | Mild | Normal | Strong | 1 |
| 12 | Overcast | Mild | High | Strong | 1 |
| 13 | Overcast | Hot | Normal | Weak | 1 |
| 14 | Rain | Mild | High | Strong | 0 |
@@ -238,7 +245,7 @@ os.system(cmd)
-The example we will look at is a classical one in many Machine -Learning applications. Based on various meteorological features, we -have several so-called attributes which decide whether we at the end -will do some outdoor activity like skiing, going for a bike ride etc -etc. The table here contains the feautures outlook, temperature, -humidity and wind. The target or output is whether we ride -(True=1) or whether we do something else that day (False=0). The -attributes for each feature are then sunny, overcast and rain for the -outlook, hot, cold and mild for temperature, high and normal for -humidity and weak and strong for wind. -
-The table here summarizes the various attributes and + +
# Common imports
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from IPython.display import Image
+from pydot import graph_from_dot_data
+import os
-
-
-
-
-Day Outlook Temperature Humidity Wind Ride 1 Sunny Hot High Weak 0 2 Sunny Hot High Strong 1 3 Overcast Hot High Weak 1 4 Rain Mild High Weak 1 5 Rain Cool Normal Weak 1 6 Rain Cool Normal Strong 0 7 Overcast Cool Normal Strong 1 8 Sunny Mild High Weak 0 9 Sunny Cool Normal Weak 1 10 Rain Mild Normal Weak 1 11 Sunny Mild Normal Strong 1 12 Overcast Mild High Strong 1 13 Overcast Hot Normal Weak 1 14 Rain Mild High Strong 0
-
-
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+ os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+ os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+ os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+ return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+ return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+ plt.savefig(image_path(fig_id) + ".png", format='png')
+
+infile = open(data_path("ride.csv"),'r')
+
+# Read the experimental data with Pandas
+from IPython.display import display
+ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
+ridedata = pd.DataFrame(ridedata)
+display(ridedata)
+# Features and targets
+X = ridedata.loc[:, ridedata.columns != 'Ride'].values
+display(X)
+y = ridedata.loc[:, ridedata.columns == 'Ride'].values
+display(y)
+# Categorical variables to one-hot's
+onehotencoder = OneHotEncoder(categories="auto")
+
+X = ColumnTransformer([("", onehotencoder)]).fit_transform(X)
+y.shape
+
+display(X)
+display(y)
+
@@ -251,7 +262,7 @@ The table here summarizes the various attributes and
+The above functions (gini, entropy and misclassification error) are +important components of the so-called CART algorithm. We will discuss +this algorithm below after we have discussed the information gain +algorithm ID3. + +
+In the example here we have converted all our attributes into numerical values \( 0,1,2 \) etc.
-
# Common imports
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-from sklearn.preprocessing import StandardScaler, OneHotEncoder
-from sklearn.compose import ColumnTransformer
-from IPython.display import Image
-from pydot import graph_from_dot_data
-import os
+# Split a dataset based on an attribute and an attribute value
+def test_split(index, value, dataset):
+ left, right = list(), list()
+ for row in dataset:
+ if row[index] < value:
+ left.append(row)
+ else:
+ right.append(row)
+ return left, right
+
+# Calculate the Gini index for a split dataset
+def gini_index(groups, classes):
+ # count all samples at split point
+ n_instances = float(sum([len(group) for group in groups]))
+ # sum weighted Gini index for each group
+ gini = 0.0
+ for group in groups:
+ size = float(len(group))
+ # avoid divide by zero
+ if size == 0:
+ continue
+ score = 0.0
+ # score the group based on the score for each class
+ for class_val in classes:
+ p = [row[-1] for row in group].count(class_val) / size
+ score += p * p
+ # weight the group score by its relative size
+ gini += (1.0 - score) * (size / n_instances)
+ return gini
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
+# Select the best split point for a dataset
+def get_split(dataset):
+ class_values = list(set(row[-1] for row in dataset))
+ b_index, b_value, b_score, b_groups = 999, 999, 999, None
+ for index in range(len(dataset[0])-1):
+ for row in dataset:
+ groups = test_split(index, row[index], dataset)
+ gini = gini_index(groups, class_values)
+ print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini))
+ if gini < b_score:
+ b_index, b_value, b_score, b_groups = index, row[index], gini, groups
+ return {'index':b_index, 'value':b_value, 'groups':b_groups}
+
+dataset = [[0,0,0,0,0],
+ [0,0,0,1,1],
+ [1,0,0,0,1],
+ [2,1,0,0,1],
+ [2,2,1,0,1],
+ [2,2,1,1,0],
+ [1,2,1,1,1],
+ [0,1,0,0,0],
+ [0,2,1,0,1],
+ [2,1,1,0,1],
+ [0,1,1,1,1],
+ [1,1,0,1,1],
+ [1,0,1,0,1],
+ [2,1,0,1,0]]
-if not os.path.exists(PROJECT_ROOT_DIR):
- os.mkdir(PROJECT_ROOT_DIR)
-
-if not os.path.exists(FIGURE_ID):
- os.makedirs(FIGURE_ID)
-
-if not os.path.exists(DATA_ID):
- os.makedirs(DATA_ID)
-
-def image_path(fig_id):
- return os.path.join(FIGURE_ID, fig_id)
-
-def data_path(dat_id):
- return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
- plt.savefig(image_path(fig_id) + ".png", format='png')
-
-infile = open(data_path("ride.csv"),'r')
-
-# Read the experimental data with Pandas
-from IPython.display import display
-ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
-ridedata = pd.DataFrame(ridedata)
-display(ridedata)
-# Features and targets
-X = ridedata.loc[:, ridedata.columns != 'Ride'].values
-display(X)
-y = ridedata.loc[:, ridedata.columns == 'Ride'].values
-display(y)
-# Categorical variables to one-hot's
-onehotencoder = OneHotEncoder(categories="auto")
-
-X = ColumnTransformer([("", onehotencoder)]).fit_transform(X)
-y.shape
-
-display(X)
-display(y)
+split = get_split(dataset)
+print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
@@ -268,7 +278,7 @@ display(y)
-The above functions (gini, entropy and misclassification error) are -important components of the so-called CART algorithm. We will discuss -this algorithm below after we have discussed the information gain -algorithm ID3. +ID3, learns decision trees by constructing +them topdown, beginning with the question which attribute should be tested at the root of the tree? + +
-In the example here we have converted all our attributes into numerical values \( 0,1,2 \) etc. +We would like to select the attribute that is most useful for classifying +examples.
+What is a good quantitative measure of the worth of an attribute? - -
# Split a dataset based on an attribute and an attribute value
-def test_split(index, value, dataset):
- left, right = list(), list()
- for row in dataset:
- if row[index] < value:
- left.append(row)
- else:
- right.append(row)
- return left, right
-
-# Calculate the Gini index for a split dataset
-def gini_index(groups, classes):
- # count all samples at split point
- n_instances = float(sum([len(group) for group in groups]))
- # sum weighted Gini index for each group
- gini = 0.0
- for group in groups:
- size = float(len(group))
- # avoid divide by zero
- if size == 0:
- continue
- score = 0.0
- # score the group based on the score for each class
- for class_val in classes:
- p = [row[-1] for row in group].count(class_val) / size
- score += p * p
- # weight the group score by its relative size
- gini += (1.0 - score) * (size / n_instances)
- return gini
+
+Information gain measures how well a given attribute separates the
+training examples according to their target classification.
-# Select the best split point for a dataset
-def get_split(dataset):
- class_values = list(set(row[-1] for row in dataset))
- b_index, b_value, b_score, b_groups = 999, 999, 999, None
- for index in range(len(dataset[0])-1):
- for row in dataset:
- groups = test_split(index, row[index], dataset)
- gini = gini_index(groups, class_values)
- print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini))
- if gini < b_score:
- b_index, b_value, b_score, b_groups = index, row[index], gini, groups
- return {'index':b_index, 'value':b_value, 'groups':b_groups}
-
-dataset = [[0,0,0,0,0],
- [0,0,0,1,1],
- [1,0,0,0,1],
- [2,1,0,0,1],
- [2,2,1,0,1],
- [2,2,1,1,0],
- [1,2,1,1,1],
- [0,1,0,0,0],
- [0,2,1,0,1],
- [2,1,1,0,1],
- [0,1,1,1,1],
- [1,1,0,1,1],
- [1,0,1,0,1],
- [2,1,0,1,0]]
+
+The ID3 algorithm uses this information gain measure to select among the candidate
+attributes at each step while growing the tree.
-split = get_split(dataset)
-print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
-
@@ -284,7 +236,7 @@ split = get_split(dataset)
-ID3, learns decision trees by constructing -them topdown, beginning with the question which attribute should be tested at the root of the tree? -
import re
+import math
+from collections import deque
-The ID3 algorithm selects, which attribute to test at each node in the
-tree.
+# x is examples in training set
+# y is set of targets
+# label is target attributes
+# Node is a class which has properties values, childs, and next
+# root is top node in the decision tree
-
-We would like to select the attribute that is most useful for classifying
-examples.
+class Node(object):
+ def __init__(self):
+ self.value = None
+ self.next = None
+ self.childs = None
-
-What is a good quantitative measure of the worth of an attribute?
+# Simple class of Decision Tree
+# Aimed for who want to learn Decision Tree, so it is not optimized
+class DecisionTree(object):
+ def __init__(self, sample, attributes, labels):
+ self.sample = sample
+ self.attributes = attributes
+ self.labels = labels
+ self.labelCodes = None
+ self.labelCodesCount = None
+ self.initLabelCodes()
+ # print(self.labelCodes)
+ self.root = None
+ self.entropy = self.getEntropy([x for x in range(len(self.labels))])
-
-Information gain measures how well a given attribute separates the
-training examples according to their target classification.
+ def initLabelCodes(self):
+ self.labelCodes = []
+ self.labelCodesCount = []
+ for l in self.labels:
+ if l not in self.labelCodes:
+ self.labelCodes.append(l)
+ self.labelCodesCount.append(0)
+ self.labelCodesCount[self.labelCodes.index(l)] += 1
-
-The ID3 algorithm uses this information gain measure to select among the candidate
-attributes at each step while growing the tree.
+ def getLabelCodeId(self, sampleId):
+ return self.labelCodes.index(self.labels[sampleId])
+ def getAttributeValues(self, sampleIds, attributeId):
+ vals = []
+ for sid in sampleIds:
+ val = self.sample[sid][attributeId]
+ if val not in vals:
+ vals.append(val)
+ # print(vals)
+ return vals
+
+ def getEntropy(self, sampleIds):
+ entropy = 0
+ labelCount = [0] * len(self.labelCodes)
+ for sid in sampleIds:
+ labelCount[self.getLabelCodeId(sid)] += 1
+ # print("-ge", labelCount)
+ for lv in labelCount:
+ # print(lv)
+ if lv != 0:
+ entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
+ else:
+ entropy += 0
+ return entropy
+
+ def getDominantLabel(self, sampleIds):
+ labelCodesCount = [0] * len(self.labelCodes)
+ for sid in sampleIds:
+ labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
+ return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
+
+ def getInformationGain(self, sampleIds, attributeId):
+ gain = self.getEntropy(sampleIds)
+ attributeVals = []
+ attributeValsCount = []
+ attributeValsIds = []
+ for sid in sampleIds:
+ val = self.sample[sid][attributeId]
+ if val not in attributeVals:
+ attributeVals.append(val)
+ attributeValsCount.append(0)
+ attributeValsIds.append([])
+ vid = attributeVals.index(val)
+ attributeValsCount[vid] += 1
+ attributeValsIds[vid].append(sid)
+ # print("-gig", self.attributes[attributeId])
+ for vc, vids in zip(attributeValsCount, attributeValsIds):
+ # print("-gig", vids)
+ gain -= vc/len(sampleIds) * self.getEntropy(vids)
+ return gain
+
+ def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
+ attributesEntropy = [0] * len(attributeIds)
+ for i, attId in zip(range(len(attributeIds)), attributeIds):
+ attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
+ maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
+ return self.attributes[maxId], maxId
+
+ def isSingleLabeled(self, sampleIds):
+ label = self.labels[sampleIds[0]]
+ for sid in sampleIds:
+ if self.labels[sid] != label:
+ return False
+ return True
+
+ def getLabel(self, sampleId):
+ return self.labels[sampleId]
+
+ def id3(self):
+ sampleIds = [x for x in range(len(self.sample))]
+ attributeIds = [x for x in range(len(self.attributes))]
+ self.root = self.id3Recv(sampleIds, attributeIds, self.root)
+
+ def id3Recv(self, sampleIds, attributeIds, root):
+ root = Node() # Initialize current root
+ if self.isSingleLabeled(sampleIds):
+ root.value = self.labels[sampleIds[0]]
+ return root
+ # print(attributeIds)
+ if len(attributeIds) == 0:
+ root.value = self.getDominantLabel(sampleIds)
+ return root
+ bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
+ sampleIds, attributeIds)
+ # print(bestAttrName)
+ root.value = bestAttrName
+ root.childs = [] # Create list of children
+ for value in self.getAttributeValues(sampleIds, bestAttrId):
+ # print(value)
+ child = Node()
+ child.value = value
+ root.childs.append(child) # Append new child node to current
+ # root
+ childSampleIds = []
+ for sid in sampleIds:
+ if self.sample[sid][bestAttrId] == value:
+ childSampleIds.append(sid)
+ if len(childSampleIds) == 0:
+ child.next = self.getDominantLabel(sampleIds)
+ else:
+ # print(bestAttrName, bestAttrId)
+ # print(attributeIds)
+ if len(attributeIds) > 0 and bestAttrId in attributeIds:
+ toRemove = attributeIds.index(bestAttrId)
+ attributeIds.pop(toRemove)
+ child.next = self.id3Recv(
+ childSampleIds, attributeIds, child.next)
+ return root
+
+ def printTree(self):
+ if self.root:
+ roots = deque()
+ roots.append(self.root)
+ while len(roots) > 0:
+ root = roots.popleft()
+ print(root.value)
+ if root.childs:
+ for child in root.childs:
+ print('({})'.format(child.value))
+ roots.append(child.next)
+ elif root.next:
+ print(root.next)
+
+
+def test():
+ f = open('DataFiles/rideclass.csv')
+ attributes = f.readline().split(',')
+ attributes = attributes[1:len(attributes)-1]
+ print(attributes)
+ sample = f.readlines()
+ f.close()
+ for i in range(len(sample)):
+ sample[i] = re.sub('\d+,', '', sample[i])
+ sample[i] = sample[i].strip().split(',')
+ labels = []
+ for s in sample:
+ labels.append(s.pop())
+ # print(sample)
+ # print(labels)
+ decisionTree = DecisionTree(sample, attributes, labels)
+ print("System entropy {}".format(decisionTree.entropy))
+ decisionTree.id3()
+ decisionTree.printTree()
+
+
+if __name__ == '__main__':
+ test()
+
@@ -242,7 +396,7 @@ attributes at each step while growing the tree.
-import re -import math -from collections import deque -
- - - - - + +
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+from sklearn.svm import SVC
+from sklearn.linear_model import LogisticRegression
+from sklearn.tree import DecisionTreeClassifier
-
-class Node(object):
- def __init__(self):
- self.value = None
- self.next = None
- self.childs = None
-
-
-
-
-class DecisionTree(object):
- def __init__(self, sample, attributes, labels):
- self.sample = sample
- self.attributes = attributes
- self.labels = labels
- self.labelCodes = None
- self.labelCodesCount = None
- self.initLabelCodes()
- # print(self.labelCodes)
- self.root = None
- self.entropy = self.getEntropy([x for x in range(len(self.labels))])
-
-
- def initLabelCodes(self):
- self.labelCodes = []
- self.labelCodesCount = []
- for l in self.labels:
- if l not in self.labelCodes:
- self.labelCodes.append(l)
- self.labelCodesCount.append(0)
- self.labelCodesCount[self.labelCodes.index(l)] += 1
-
-
- def getLabelCodeId(self, sampleId):
- return self.labelCodes.index(self.labels[sampleId])
-
-
- def getAttributeValues(self, sampleIds, attributeId):
- vals = []
- for sid in sampleIds:
- val = self.sample[sid][attributeId]
- if val not in vals:
- vals.append(val)
- # print(vals)
- return vals
-
-
- def getEntropy(self, sampleIds):
- entropy = 0
- labelCount = [0] * len(self.labelCodes)
- for sid in sampleIds:
- labelCount[self.getLabelCodeId(sid)] += 1
- # print("-ge", labelCount)
- for lv in labelCount:
- # print(lv)
- if lv != 0:
- entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
- else:
- entropy += 0
- return entropy
-
-
- def getDominantLabel(self, sampleIds):
- labelCodesCount = [0] * len(self.labelCodes)
- for sid in sampleIds:
- labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
- return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
-
-
- def getInformationGain(self, sampleIds, attributeId):
- gain = self.getEntropy(sampleIds)
- attributeVals = []
- attributeValsCount = []
- attributeValsIds = []
- for sid in sampleIds:
- val = self.sample[sid][attributeId]
- if val not in attributeVals:
- attributeVals.append(val)
- attributeValsCount.append(0)
- attributeValsIds.append([])
- vid = attributeVals.index(val)
- attributeValsCount[vid] += 1
- attributeValsIds[vid].append(sid)
- # print("-gig", self.attributes[attributeId])
- for vc, vids in zip(attributeValsCount, attributeValsIds):
- # print("-gig", vids)
- gain -= vc/len(sampleIds) * self.getEntropy(vids)
- return gain
-
-
- def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
- attributesEntropy = [0] * len(attributeIds)
- for i, attId in zip(range(len(attributeIds)), attributeIds):
- attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
- maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
- return self.attributes[maxId], maxId
-
-
- def isSingleLabeled(self, sampleIds):
- label = self.labels[sampleIds[0]]
- for sid in sampleIds:
- if self.labels[sid] != label:
- return False
- return True
-
-
- def getLabel(self, sampleId):
- return self.labels[sampleId]
-
-
- def id3(self):
- sampleIds = [x for x in range(len(self.sample))]
- attributeIds = [x for x in range(len(self.attributes))]
- self.root = self.id3Recv(sampleIds, attributeIds, self.root)
-
-
- def id3Recv(self, sampleIds, attributeIds, root):
- root = Node() # Initialize current root
- if self.isSingleLabeled(sampleIds):
- root.value = self.labels[sampleIds[0]]
- return root
- # print(attributeIds)
- if len(attributeIds) == 0:
- root.value = self.getDominantLabel(sampleIds)
- return root
- bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
- sampleIds, attributeIds)
- # print(bestAttrName)
- root.value = bestAttrName
- root.childs = [] # Create list of children
- for value in self.getAttributeValues(sampleIds, bestAttrId):
- # print(value)
- child = Node()
- child.value = value
- root.childs.append(child) # Append new child node to current
- # root
- childSampleIds = []
- for sid in sampleIds:
- if self.sample[sid][bestAttrId] == value:
- childSampleIds.append(sid)
- if len(childSampleIds) == 0:
- child.next = self.getDominantLabel(sampleIds)
- else:
- # print(bestAttrName, bestAttrId)
- # print(attributeIds)
- if len(attributeIds) > 0 and bestAttrId in attributeIds:
- toRemove = attributeIds.index(bestAttrId)
- attributeIds.pop(toRemove)
- child.next = self.id3Recv(
- childSampleIds, attributeIds, child.next)
- return root
-
-
- def printTree(self):
- if self.root:
- roots = deque()
- roots.append(self.root)
- while len(roots) > 0:
- root = roots.popleft()
- print(root.value)
- if root.childs:
- for child in root.childs:
- print('({})'.format(child.value))
- roots.append(child.next)
- elif root.next:
- print(root.next)
-
-
-def test():
- f = open('DataFiles/rideclass.csv')
- attributes = f.readline().split(',')
- attributes = attributes[1:len(attributes)-1]
- print(attributes)
- sample = f.readlines()
- f.close()
- for i in range(len(sample)):
- sample[i] = re.sub('\d+,', '', sample[i])
- sample[i] = sample[i].strip().split(',')
- labels = []
- for s in sample:
- labels.append(s.pop())
- # print(sample)
- # print(labels)
- decisionTree = DecisionTree(sample, attributes, labels)
- print("System entropy {}".format(decisionTree.entropy))
- decisionTree.id3()
- decisionTree.printTree()
-
-
-if __name__ == '__main__':
- test()
+# Load the data
+cancer = load_breast_cancer()
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+# Logistic Regression
+logreg = LogisticRegression(solver='lbfgs')
+logreg.fit(X_train, y_train)
+print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
+# Support vector machine
+svm = SVC(gamma='auto', C=100)
+svm.fit(X_train, y_train)
+print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
+# Decision Trees
+deep_tree_clf = DecisionTreeClassifier(max_depth=None)
+deep_tree_clf.fit(X_train, y_train)
+print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+# Logistic Regression
+logreg.fit(X_train_scaled, y_train)
+print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Support Vector Machine
+svm.fit(X_train_scaled, y_train)
+print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Decision Trees
+deep_tree_clf.fit(X_train_scaled, y_train)
+print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
+
@@ -415,7 +249,7 @@ if __name__ == '__main__':
-
import matplotlib.pyplot as plt
+from __future__ import division, print_function, unicode_literals
+
+# Common imports
import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn.datasets import load_breast_cancer
+import os
+
+# to make this notebook's output stable across runs
+np.random.seed(42)
+
+# To plot pretty figures
+import matplotlib
+import matplotlib.pyplot as plt
+from matplotlib.colors import ListedColormap
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
from sklearn.svm import SVC
-from sklearn.linear_model import LogisticRegression
+from sklearn import datasets
from sklearn.tree import DecisionTreeClassifier
+from sklearn.datasets import make_moons
+from sklearn.tree import export_graphviz
-# Load the data
-cancer = load_breast_cancer()
+Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)
-X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-print(X_train.shape)
-print(X_test.shape)
-# Logistic Regression
-logreg = LogisticRegression(solver='lbfgs')
-logreg.fit(X_train, y_train)
-print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
-# Support vector machine
-svm = SVC(gamma='auto', C=100)
-svm.fit(X_train, y_train)
-print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
-# Decision Trees
-deep_tree_clf = DecisionTreeClassifier(max_depth=None)
-deep_tree_clf.fit(X_train, y_train)
-print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
-#now scale the data
-from sklearn.preprocessing import StandardScaler
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-# Logistic Regression
-logreg.fit(X_train_scaled, y_train)
-print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Support Vector Machine
-svm.fit(X_train_scaled, y_train)
-print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Decision Trees
-deep_tree_clf.fit(X_train_scaled, y_train)
-print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
+deep_tree_clf1 = DecisionTreeClassifier(random_state=42)
+deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)
+deep_tree_clf1.fit(Xm, ym)
+deep_tree_clf2.fit(Xm, ym)
+
+
+def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):
+ x1s = np.linspace(axes[0], axes[1], 100)
+ x2s = np.linspace(axes[2], axes[3], 100)
+ x1, x2 = np.meshgrid(x1s, x2s)
+ X_new = np.c_[x1.ravel(), x2.ravel()]
+ y_pred = clf.predict(X_new).reshape(x1.shape)
+ custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
+ plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
+ if not iris:
+ custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
+ plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
+ if plot_training:
+ plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa")
+ plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor")
+ plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica")
+ plt.axis(axes)
+ if iris:
+ plt.xlabel("Petal length", fontsize=14)
+ plt.ylabel("Petal width", fontsize=14)
+ else:
+ plt.xlabel(r"$x_1$", fontsize=18)
+ plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
+ if legend:
+ plt.legend(loc="lower right", fontsize=14)
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
+plt.title("No restrictions", fontsize=16)
+plt.subplot(122)
+plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
+plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14)
+plt.show()
@@ -255,7 +272,7 @@ deep_tree_clf.fit(X_train_scaled, y_train)
-
from __future__ import division, print_function, unicode_literals
+np.random.seed(6)
+Xs = np.random.rand(100, 2) - 0.5
+ys = (Xs[:, 0] > 0).astype(np.float32) * 2
-# Common imports
-import numpy as np
-import os
+angle = np.pi/4
+rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
+Xsr = Xs.dot(rotation_matrix)
-# to make this notebook's output stable across runs
-np.random.seed(42)
+tree_clf_s = DecisionTreeClassifier(random_state=42)
+tree_clf_s.fit(Xs, ys)
+tree_clf_sr = DecisionTreeClassifier(random_state=42)
+tree_clf_sr.fit(Xsr, ys)
-# To plot pretty figures
-import matplotlib
-import matplotlib.pyplot as plt
-from matplotlib.colors import ListedColormap
-plt.rcParams['axes.labelsize'] = 14
-plt.rcParams['xtick.labelsize'] = 12
-plt.rcParams['ytick.labelsize'] = 12
-
-
-from sklearn.svm import SVC
-from sklearn import datasets
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.datasets import make_moons
-from sklearn.tree import export_graphviz
-
-Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)
-
-deep_tree_clf1 = DecisionTreeClassifier(random_state=42)
-deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)
-deep_tree_clf1.fit(Xm, ym)
-deep_tree_clf2.fit(Xm, ym)
-
-
-def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):
- x1s = np.linspace(axes[0], axes[1], 100)
- x2s = np.linspace(axes[2], axes[3], 100)
- x1, x2 = np.meshgrid(x1s, x2s)
- X_new = np.c_[x1.ravel(), x2.ravel()]
- y_pred = clf.predict(X_new).reshape(x1.shape)
- custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
- plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
- if not iris:
- custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
- plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
- if plot_training:
- plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa")
- plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor")
- plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica")
- plt.axis(axes)
- if iris:
- plt.xlabel("Petal length", fontsize=14)
- plt.ylabel("Petal width", fontsize=14)
- else:
- plt.xlabel(r"$x_1$", fontsize=18)
- plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
- if legend:
- plt.legend(loc="lower right", fontsize=14)
plt.figure(figsize=(11, 4))
plt.subplot(121)
-plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
-plt.title("No restrictions", fontsize=16)
+plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
plt.subplot(122)
-plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
-plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14)
+plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+
plt.show()
@@ -278,7 +228,7 @@ plt.show()
-
np.random.seed(6)
-Xs = np.random.rand(100, 2) - 0.5
-ys = (Xs[:, 0] > 0).astype(np.float32) * 2
+# Quadratic training set + noise
+np.random.seed(42)
+m = 200
+X = np.random.rand(m, 1)
+y = 4 * (X - 0.5) ** 2
+y = y + np.random.randn(m, 1) / 10
+
+
-angle = np.pi/4
-rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
-Xsr = Xs.dot(rotation_matrix)
+
+
from sklearn.tree import DecisionTreeRegressor
-tree_clf_s = DecisionTreeClassifier(random_state=42)
-tree_clf_s.fit(Xs, ys)
-tree_clf_sr = DecisionTreeClassifier(random_state=42)
-tree_clf_sr.fit(Xsr, ys)
-
-plt.figure(figsize=(11, 4))
-plt.subplot(121)
-plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
-plt.subplot(122)
-plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
-
-plt.show()
+tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg.fit(X, y)
@@ -234,7 +222,7 @@ plt.show()
- - -
# Quadratic training set + noise
-np.random.seed(42)
-m = 200
-X = np.random.rand(m, 1)
-y = 4 * (X - 0.5) ** 2
-y = y + np.random.randn(m, 1) / 10
-
from sklearn.tree import DecisionTreeRegressor
-tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg.fit(X, y)
+tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)
+tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)
+tree_reg1.fit(X, y)
+tree_reg2.fit(X, y)
+
+def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel="$y$"):
+ x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)
+ y_pred = tree_reg.predict(x1)
+ plt.axis(axes)
+ plt.xlabel("$x_1$", fontsize=18)
+ if ylabel:
+ plt.ylabel(ylabel, fontsize=18, rotation=0)
+ plt.plot(X, y, "b.")
+ plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$")
+
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_regression_predictions(tree_reg1, X, y)
+for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
+ plt.plot([split, split], [-0.2, 1], style, linewidth=2)
+plt.text(0.21, 0.65, "Depth=0", fontsize=15)
+plt.text(0.01, 0.2, "Depth=1", fontsize=13)
+plt.text(0.65, 0.8, "Depth=1", fontsize=13)
+plt.legend(loc="upper center", fontsize=18)
+plt.title("max_depth=2", fontsize=14)
+
+plt.subplot(122)
+plot_regression_predictions(tree_reg2, X, y, ylabel=None)
+for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
+ plt.plot([split, split], [-0.2, 1], style, linewidth=2)
+for split in (0.0458, 0.1298, 0.2873, 0.9040):
+ plt.plot([split, split], [-0.2, 1], "k:", linewidth=1)
+plt.text(0.3, 0.5, "Depth=2", fontsize=13)
+plt.title("max_depth=3", fontsize=14)
+
+plt.show()
++ + +
tree_reg1 = DecisionTreeRegressor(random_state=42)
+tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)
+tree_reg1.fit(X, y)
+tree_reg2.fit(X, y)
+
+x1 = np.linspace(0, 1, 500).reshape(-1, 1)
+y_pred1 = tree_reg1.predict(x1)
+y_pred2 = tree_reg2.predict(x1)
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.plot(X, y, "b.")
+plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$")
+plt.axis([0, 1, -0.2, 1.1])
+plt.xlabel("$x_1$", fontsize=18)
+plt.ylabel("$y$", fontsize=18, rotation=0)
+plt.legend(loc="upper center", fontsize=18)
+plt.title("No restrictions", fontsize=14)
+
+plt.subplot(122)
+plt.plot(X, y, "b.")
+plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$")
+plt.axis([0, 1, -0.2, 1.1])
+plt.xlabel("$x_1$", fontsize=18)
+plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14)
+
+plt.show()
@@ -228,7 +278,7 @@ tree_reg.fit(X, y)
+
from sklearn.tree import DecisionTreeRegressor
+- - -
tree_reg1 = DecisionTreeRegressor(random_state=42)
-tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)
-tree_reg1.fit(X, y)
-tree_reg2.fit(X, y)
-
-x1 = np.linspace(0, 1, 500).reshape(-1, 1)
-y_pred1 = tree_reg1.predict(x1)
-y_pred2 = tree_reg2.predict(x1)
-
-plt.figure(figsize=(11, 4))
-
-plt.subplot(121)
-plt.plot(X, y, "b.")
-plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$")
-plt.axis([0, 1, -0.2, 1.1])
-plt.xlabel("$x_1$", fontsize=18)
-plt.ylabel("$y$", fontsize=18, rotation=0)
-plt.legend(loc="upper center", fontsize=18)
-plt.title("No restrictions", fontsize=14)
-
-plt.subplot(122)
-plt.plot(X, y, "b.")
-plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$")
-plt.axis([0, 1, -0.2, 1.1])
-plt.xlabel("$x_1$", fontsize=18)
-plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14)
-
-plt.show()
-
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs028.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs028.html index de920c9e4..8646025cb 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs028.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs028.html @@ -47,52 +47,47 @@ Automatically generated HTML file from DocOnce source 2, None, '___sec1'), - ('A typical Decision Tree with its pertinent Jargon, Regeression ' - 'Problem', - 2, - None, - '___sec2'), - ('General Features', 2, None, '___sec3'), - ('How do we set it up?', 2, None, '___sec4'), - ('Decision trees and Regression', 2, None, '___sec5'), - ('Building a tree, regression', 2, None, '___sec6'), + ('General Features', 2, None, '___sec2'), + ('How do we set it up?', 2, None, '___sec3'), + ('Decision trees and Regression', 2, None, '___sec4'), + ('Building a tree, regression', 2, None, '___sec5'), ('A top-down approach, recursive binary splitting', 2, None, - '___sec7'), - ('Making a tree', 2, None, '___sec8'), - ('Pruning the tree', 2, None, '___sec9'), - ('Cost complexity pruning', 2, None, '___sec10'), - ('Schematic Regression Procedure', 2, None, '___sec11'), - ('A Classification Tree', 2, None, '___sec12'), - ('Growing a classification tree', 2, None, '___sec13'), - ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Visualizing the Tree, Classification', 2, None, '___sec15'), - ('Visualizing the Tree, The Moons', 2, None, '___sec16'), - ('Computing the Gini index', 2, None, '___sec17'), - ('Simple Python Code to read in Data', 2, None, '___sec18'), - ('Computing the Gini Factor', 2, None, '___sec19'), - ('Entropy and the ID3 algorithm', 2, None, '___sec20'), - ('Implementing the ID3 Algorithm', 2, None, '___sec21'), + '___sec6'), + ('Making a tree', 2, None, '___sec7'), + ('Pruning the tree', 2, None, '___sec8'), + ('Cost complexity pruning', 2, None, '___sec9'), + ('Schematic Regression Procedure', 2, None, '___sec10'), + ('A Classification Tree', 2, None, '___sec11'), + ('Growing a classification tree', 2, None, '___sec12'), + ('Classification tree, how to split nodes', 2, None, '___sec13'), + ('Visualizing the Tree, Classification', 2, None, '___sec14'), + ('Visualizing the Tree, The Moons', 2, None, '___sec15'), + ('Computing the Gini index', 2, None, '___sec16'), + ('Simple Python Code to read in Data', 2, None, '___sec17'), + ('Computing the Gini Factor', 2, None, '___sec18'), + ('Entropy and the ID3 algorithm', 2, None, '___sec19'), + ('Implementing the ID3 Algorithm', 2, None, '___sec20'), ('Cancer Data again now with Decision Trees and other Methods', 2, None, - '___sec22'), - ('Another example, the moons again', 2, None, '___sec23'), - ('Playing around with regions', 2, None, '___sec24'), - ('Regression trees', 2, None, '___sec25'), - ('Final regressor code', 2, None, '___sec26'), - ('Pros and cons of trees, pros', 2, None, '___sec27'), - ('Disadvantages', 2, None, '___sec28'), - ('Bagging', 2, None, '___sec29'), - ('More bagging', 2, None, '___sec30'), - ('Simple example, head or tail', 2, None, '___sec31'), - ('Bagging Example', 2, None, '___sec32'), - ('Random forests', 2, None, '___sec33'), - ('A simple scikit-learn example', 2, None, '___sec34'), - ('Please, not the moons again!', 2, None, '___sec35'), - ('Bagging examples', 2, None, '___sec36'), - ('Then random forests', 2, None, '___sec37')]} + '___sec21'), + ('Another example, the moons again', 2, None, '___sec22'), + ('Playing around with regions', 2, None, '___sec23'), + ('Regression trees', 2, None, '___sec24'), + ('Final regressor code', 2, None, '___sec25'), + ('Pros and cons of trees, pros', 2, None, '___sec26'), + ('Disadvantages', 2, None, '___sec27'), + ('Bagging', 2, None, '___sec28'), + ('More bagging', 2, None, '___sec29'), + ('Simple example, head or tail', 2, None, '___sec30'), + ('Bagging Example', 2, None, '___sec31'), + ('Random forests', 2, None, '___sec32'), + ('A simple scikit-learn example', 2, None, '___sec33'), + ('Please, not the moons again!', 2, None, '___sec34'), + ('Bagging examples', 2, None, '___sec35'), + ('Then random forests', 2, None, '___sec36')]} end of tocinfo --> @@ -132,42 +127,41 @@ MathJax.Hub.Config({ @@ -183,18 +177,21 @@ MathJax.Hub.Config({ -
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs029.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs029.html index 0ccfacfb4..351fcd155 100644 --- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs029.html +++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs029.html @@ -47,52 +47,47 @@ Automatically generated HTML file from DocOnce source 2, None, '___sec1'), - ('A typical Decision Tree with its pertinent Jargon, Regeression ' - 'Problem', - 2, - None, - '___sec2'), - ('General Features', 2, None, '___sec3'), - ('How do we set it up?', 2, None, '___sec4'), - ('Decision trees and Regression', 2, None, '___sec5'), - ('Building a tree, regression', 2, None, '___sec6'), + ('General Features', 2, None, '___sec2'), + ('How do we set it up?', 2, None, '___sec3'), + ('Decision trees and Regression', 2, None, '___sec4'), + ('Building a tree, regression', 2, None, '___sec5'), ('A top-down approach, recursive binary splitting', 2, None, - '___sec7'), - ('Making a tree', 2, None, '___sec8'), - ('Pruning the tree', 2, None, '___sec9'), - ('Cost complexity pruning', 2, None, '___sec10'), - ('Schematic Regression Procedure', 2, None, '___sec11'), - ('A Classification Tree', 2, None, '___sec12'), - ('Growing a classification tree', 2, None, '___sec13'), - ('Classification tree, how to split nodes', 2, None, '___sec14'), - ('Visualizing the Tree, Classification', 2, None, '___sec15'), - ('Visualizing the Tree, The Moons', 2, None, '___sec16'), - ('Computing the Gini index', 2, None, '___sec17'), - ('Simple Python Code to read in Data', 2, None, '___sec18'), - ('Computing the Gini Factor', 2, None, '___sec19'), - ('Entropy and the ID3 algorithm', 2, None, '___sec20'), - ('Implementing the ID3 Algorithm', 2, None, '___sec21'), + '___sec6'), + ('Making a tree', 2, None, '___sec7'), + ('Pruning the tree', 2, None, '___sec8'), + ('Cost complexity pruning', 2, None, '___sec9'), + ('Schematic Regression Procedure', 2, None, '___sec10'), + ('A Classification Tree', 2, None, '___sec11'), + ('Growing a classification tree', 2, None, '___sec12'), + ('Classification tree, how to split nodes', 2, None, '___sec13'), + ('Visualizing the Tree, Classification', 2, None, '___sec14'), + ('Visualizing the Tree, The Moons', 2, None, '___sec15'), + ('Computing the Gini index', 2, None, '___sec16'), + ('Simple Python Code to read in Data', 2, None, '___sec17'), + ('Computing the Gini Factor', 2, None, '___sec18'), + ('Entropy and the ID3 algorithm', 2, None, '___sec19'), + ('Implementing the ID3 Algorithm', 2, None, '___sec20'), ('Cancer Data again now with Decision Trees and other Methods', 2, None, - '___sec22'), - ('Another example, the moons again', 2, None, '___sec23'), - ('Playing around with regions', 2, None, '___sec24'), - ('Regression trees', 2, None, '___sec25'), - ('Final regressor code', 2, None, '___sec26'), - ('Pros and cons of trees, pros', 2, None, '___sec27'), - ('Disadvantages', 2, None, '___sec28'), - ('Bagging', 2, None, '___sec29'), - ('More bagging', 2, None, '___sec30'), - ('Simple example, head or tail', 2, None, '___sec31'), - ('Bagging Example', 2, None, '___sec32'), - ('Random forests', 2, None, '___sec33'), - ('A simple scikit-learn example', 2, None, '___sec34'), - ('Please, not the moons again!', 2, None, '___sec35'), - ('Bagging examples', 2, None, '___sec36'), - ('Then random forests', 2, None, '___sec37')]} + '___sec21'), + ('Another example, the moons again', 2, None, '___sec22'), + ('Playing around with regions', 2, None, '___sec23'), + ('Regression trees', 2, None, '___sec24'), + ('Final regressor code', 2, None, '___sec25'), + ('Pros and cons of trees, pros', 2, None, '___sec26'), + ('Disadvantages', 2, None, '___sec27'), + ('Bagging', 2, None, '___sec28'), + ('More bagging', 2, None, '___sec29'), + ('Simple example, head or tail', 2, None, '___sec30'), + ('Bagging Example', 2, None, '___sec31'), + ('Random forests', 2, None, '___sec32'), + ('A simple scikit-learn example', 2, None, '___sec33'), + ('Please, not the moons again!', 2, None, '___sec34'), + ('Bagging examples', 2, None, '___sec35'), + ('Then random forests', 2, None, '___sec36')]} end of tocinfo --> @@ -132,42 +127,41 @@ MathJax.Hub.Config({ @@ -183,19 +177,21 @@ MathJax.Hub.Config({ -
+The plain decision trees suffer from high +variance. This means that if we split the training data into two parts +at random, and fit a decision tree to both halves, the results that we +get could be quite different. In contrast, a procedure with low +variance will yield similar results if applied repeatedly to distinct +data sets; linear regression tends to have low variance, if the ratio +of \( n \) to \( p \) is moderately large. -However, by aggregating many decision trees, using methods like bagging, random forests, and boosting, the predictive performance of trees can be substantially improved. +
+Bootstrap aggregation, or just bagging, is a +general-purpose procedure for reducing the variance of a statistical +learning method.
@@ -221,7 +217,6 @@ However, by aggregating many decision trees, using methods like bagging, random
-The plain decision trees suffer from high -variance. This means that if we split the training data into two parts -at random, and fit a decision tree to both halves, the results that we -get could be quite different. In contrast, a procedure with low -variance will yield similar results if applied repeatedly to distinct -data sets; linear regression tends to have low variance, if the ratio -of \( n \) to \( p \) is moderately large. +Bagging typically results in improved accuracy +over prediction using a single tree. Unfortunately, however, it can be +difficult to interpret the resulting model. Recall that one of the +advantages of decision trees is the attractive and easily interpreted +diagram that results.
-Bootstrap aggregation, or just bagging, is a -general-purpose procedure for reducing the variance of a statistical -learning method. +However, when we bag a large number of trees, it is no longer +possible to represent the resulting statistical learning procedure +using a single tree, and it is no longer clear which variables are +most important to the procedure. Thus, bagging improves prediction +accuracy at the expense of interpretability. Although the collection +of bagged trees is much more difficult to interpret than a single +tree, one can obtain an overall summary of the importance of each +predictor using the MSE (for bagging regression trees) or the Gini +index (for bagging classification trees). In the case of bagging +regression trees, we can record the total amount that the MSE is +decreased due to splits over a given predictor, averaged over all \( B \) possible +trees. A large value indicates an important predictor. Similarly, in +the context of bagging classification trees, we can add up the total +amount that the Gini index is decreased by splits over a given +predictor, averaged over all \( B \) trees.
@@ -222,7 +226,6 @@ learning method.
-Bagging typically results in improved accuracy -over prediction using a single tree. Unfortunately, however, it can be -difficult to interpret the resulting model. Recall that one of the -advantages of decision trees is the attractive and easily interpreted -diagram that results. - -
-However, when we bag a large number of trees, it is no longer -possible to represent the resulting statistical learning procedure -using a single tree, and it is no longer clear which variables are -most important to the procedure. Thus, bagging improves prediction -accuracy at the expense of interpretability. Although the collection -of bagged trees is much more difficult to interpret than a single -tree, one can obtain an overall summary of the importance of each -predictor using the MSE (for bagging regression trees) or the Gini -index (for bagging classification trees). In the case of bagging -regression trees, we can record the total amount that the MSE is -decreased due to splits over a given predictor, averaged over all \( B \) possible -trees. A large value indicates an important predictor. Similarly, in -the context of bagging classification trees, we can add up the total -amount that the Gini index is decreased by splits over a given -predictor, averaged over all \( B \) trees. + +
heads_proba = 0.51
+coin_tosses = (np.random.rand(10000, 10) < heads_proba).astype(np.int32)
+cumulative_heads_ratio = np.cumsum(coin_tosses, axis=0) / np.arange(1, 10001).reshape(-1, 1)
+plt.figure(figsize=(8,3.5))
+plt.plot(cumulative_heads_ratio)
+plt.plot([0, 10000], [0.51, 0.51], "k--", linewidth=2, label="51%")
+plt.plot([0, 10000], [0.5, 0.5], "k-", label="50%")
+plt.xlabel("Number of coin tosses")
+plt.ylabel("Heads ratio")
+plt.legend(loc="lower right")
+plt.axis([0, 10000, 0.42, 0.58])
+plt.show()
+
@@ -231,7 +216,6 @@ predictor, averaged over all \( B \) trees.
-
heads_proba = 0.51
-coin_tosses = (np.random.rand(10000, 10) < heads_proba).astype(np.int32)
-cumulative_heads_ratio = np.cumsum(coin_tosses, axis=0) / np.arange(1, 10001).reshape(-1, 1)
-plt.figure(figsize=(8,3.5))
-plt.plot(cumulative_heads_ratio)
-plt.plot([0, 10000], [0.51, 0.51], "k--", linewidth=2, label="51%")
-plt.plot([0, 10000], [0.5, 0.5], "k-", label="50%")
-plt.xlabel("Number of coin tosses")
-plt.ylabel("Heads ratio")
-plt.legend(loc="lower right")
-plt.axis([0, 10000, 0.42, 0.58])
-plt.show()
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import make_moons
+
+X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
+X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
+
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.ensemble import VotingClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.svm import SVC
+
+log_clf = LogisticRegression(solver="liblinear", random_state=42)
+rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
+svm_clf = SVC(gamma="auto", random_state=42)
+
+voting_clf = VotingClassifier(
+ estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
+ voting='hard')
+
+voting_clf.fit(X_train, y_train)
+
+from sklearn.metrics import accuracy_score
+
+for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
+ clf.fit(X_train, y_train)
+ y_pred = clf.predict(X_test)
+ print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
+
+log_clf = LogisticRegression(solver="liblinear", random_state=42)
+rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
+svm_clf = SVC(gamma="auto", probability=True, random_state=42)
+
+voting_clf = VotingClassifier(
+ estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
+ voting='soft')
+voting_clf.fit(X_train, y_train)
+
+from sklearn.metrics import accuracy_score
+
+for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
+ clf.fit(X_train, y_train)
+ y_pred = clf.predict(X_test)
+ print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
@@ -221,7 +246,6 @@ plt.show()
+Random forests provide an improvement over bagged trees by way of a +small tweak that decorrelates the trees. - -
from sklearn.model_selection import train_test_split
-from sklearn.datasets import make_moons
+
+As in bagging, we build a
+number of decision trees on bootstrapped training samples. But when
+building these decision trees, each time a split in a tree is
+considered, a random sample of \( m \) predictors is chosen as split
+candidates from the full set of \( p \) predictors. The split is allowed to
+use only one of those \( m \) predictors.
-X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
-X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
+
+A fresh sample of \( m \) predictors is
+taken at each split, and typically we choose
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.ensemble import VotingClassifier
-from sklearn.linear_model import LogisticRegression
-from sklearn.svm import SVC
+$$
+m\approx \sqrt{p}.
+$$
-log_clf = LogisticRegression(solver="liblinear", random_state=42)
-rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
-svm_clf = SVC(gamma="auto", random_state=42)
+
+In building a random forest, at
+each split in the tree, the algorithm is not even allowed to consider
+a majority of the available predictors.
-voting_clf = VotingClassifier(
- estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
- voting='hard')
+
+The reason for this is rather clever. Suppose that there is one very
+strong predictor in the data set, along with a number of other
+moderately strong predictors. Then in the collection of bagged
+variable importance random forest trees, most or all of the trees will
+use this strong predictor in the top split. Consequently, all of the
+bagged trees will look quite similar to each other. Hence the
+predictions from the bagged trees will be highly correlated.
+Unfortunately, averaging many highly correlated quantities does not
+lead to as large of a reduction in variance as averaging many
+uncorrelated quanti- ties. In particular, this means that bagging will
+not lead to a substantial reduction in variance over a single tree in
+this setting.
-voting_clf.fit(X_train, y_train)
-
-from sklearn.metrics import accuracy_score
-
-for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
- clf.fit(X_train, y_train)
- y_pred = clf.predict(X_test)
- print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
-
-log_clf = LogisticRegression(solver="liblinear", random_state=42)
-rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
-svm_clf = SVC(gamma="auto", probability=True, random_state=42)
-
-voting_clf = VotingClassifier(
- estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
- voting='soft')
-voting_clf.fit(X_train, y_train)
-
-from sklearn.metrics import accuracy_score
-
-for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
- clf.fit(X_train, y_train)
- y_pred = clf.predict(X_test)
- print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
-
@@ -251,7 +238,6 @@ voting_clf.fit(X_train, y_train)
-Random forests provide an improvement over bagged trees by way of a -small tweak that decorrelates the trees. - -
-As in bagging, we build a -number of decision trees on bootstrapped training samples. But when -building these decision trees, each time a split in a tree is -considered, a random sample of \( m \) predictors is chosen as split -candidates from the full set of \( p \) predictors. The split is allowed to -use only one of those \( m \) predictors. - -
-A fresh sample of \( m \) predictors is -taken at each split, and typically we choose - -$$ -m\approx \sqrt{p}. -$$ - -
-In building a random forest, at -each split in the tree, the algorithm is not even allowed to consider -a majority of the available predictors. - -
-The reason for this is rather clever. Suppose that there is one very -strong predictor in the data set, along with a number of other -moderately strong predictors. Then in the collection of bagged -variable importance random forest trees, most or all of the trees will -use this strong predictor in the top split. Consequently, all of the -bagged trees will look quite similar to each other. Hence the -predictions from the bagged trees will be highly correlated. -Unfortunately, averaging many highly correlated quantities does not -lead to as large of a reduction in variance as averaging many -uncorrelated quanti- ties. In particular, this means that bagging will -not lead to a substantial reduction in variance over a single tree in -this setting. + +
from sklearn.ensemble import RandomForestClassifier
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+# Data set not specificied
+X = dataset.XXX
+Y = dataset.YYY
+#Instantiate the model with 100 trees and entropy as splitting criteria
+Random_Forest_model = RandomForestClassifier(n_estimators=100,criterion="entropy")
+#Cross validation
+accuracy = cross_validate(Random_Forest_model,X,Y,cv=10)['test_score']
+
@@ -243,7 +211,6 @@ this setting.
-
from sklearn.ensemble import RandomForestClassifier
-from sklearn.preprocessing import LabelEncoder
-from sklearn.model_selection import cross_validate
-# Data set not specificied
-X = dataset.XXX
-Y = dataset.YYY
-#Instantiate the model with 100 trees and entropy as splitting criteria
-Random_Forest_model = RandomForestClassifier(n_estimators=100,criterion="entropy")
-#Cross validation
-accuracy = cross_validate(Random_Forest_model,X,Y,cv=10)['test_score']
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import make_moons
+
+X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
+X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.ensemble import VotingClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.svm import SVC
+
+log_clf = LogisticRegression(random_state=42)
+rnd_clf = RandomForestClassifier(random_state=42)
+svm_clf = SVC(random_state=42)
+
+voting_clf = VotingClassifier(
+ estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
+ voting='hard')
+voting_clf.fit(X_train, y_train)
+
+
+
+
+
from sklearn.metrics import accuracy_score
+
+for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
+ clf.fit(X_train, y_train)
+ y_pred = clf.predict(X_test)
+ print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
+
+
+
+
+
log_clf = LogisticRegression(random_state=42)
+rnd_clf = RandomForestClassifier(random_state=42)
+svm_clf = SVC(probability=True, random_state=42)
+
+voting_clf = VotingClassifier(
+ estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
+ voting='soft')
+voting_clf.fit(X_train, y_train)
+
+
+
+
+
from sklearn.metrics import accuracy_score
+
+for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
+ clf.fit(X_train, y_train)
+ y_pred = clf.predict(X_test)
+ print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
@@ -216,7 +250,6 @@ accuracy = cross_validate(Random_Forest_mode
-
from sklearn.model_selection import train_test_split
-from sklearn.datasets import make_moons
+from sklearn.ensemble import BaggingClassifier
+from sklearn.tree import DecisionTreeClassifier
-X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
-X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.ensemble import VotingClassifier
-from sklearn.linear_model import LogisticRegression
-from sklearn.svm import SVC
-
-log_clf = LogisticRegression(random_state=42)
-rnd_clf = RandomForestClassifier(random_state=42)
-svm_clf = SVC(random_state=42)
-
-voting_clf = VotingClassifier(
- estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
- voting='hard')
-voting_clf.fit(X_train, y_train)
+bag_clf = BaggingClassifier(
+ DecisionTreeClassifier(random_state=42), n_estimators=500,
+ max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)
+bag_clf.fit(X_train, y_train)
+y_pred = bag_clf.predict(X_test)
from sklearn.metrics import accuracy_score
-
-for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
- clf.fit(X_train, y_train)
- y_pred = clf.predict(X_test)
- print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
+print(accuracy_score(y_test, y_pred))
-
log_clf = LogisticRegression(random_state=42)
-rnd_clf = RandomForestClassifier(random_state=42)
-svm_clf = SVC(probability=True, random_state=42)
-
-voting_clf = VotingClassifier(
- estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
- voting='soft')
-voting_clf.fit(X_train, y_train)
+tree_clf = DecisionTreeClassifier(random_state=42)
+tree_clf.fit(X_train, y_train)
+y_pred_tree = tree_clf.predict(X_test)
+print(accuracy_score(y_test, y_pred_tree))
-
from sklearn.metrics import accuracy_score
+from matplotlib.colors import ListedColormap
-for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
- clf.fit(X_train, y_train)
- y_pred = clf.predict(X_test)
- print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
+def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):
+ x1s = np.linspace(axes[0], axes[1], 100)
+ x2s = np.linspace(axes[2], axes[3], 100)
+ x1, x2 = np.meshgrid(x1s, x2s)
+ X_new = np.c_[x1.ravel(), x2.ravel()]
+ y_pred = clf.predict(X_new).reshape(x1.shape)
+ custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
+ plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
+ if contour:
+ custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
+ plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
+ plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha)
+ plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha)
+ plt.axis(axes)
+ plt.xlabel(r"$x_1$", fontsize=18)
+ plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
+plt.figure(figsize=(11,4))
+plt.subplot(121)
+plot_decision_boundary(tree_clf, X, y)
+plt.title("Decision Tree", fontsize=14)
+plt.subplot(122)
+plot_decision_boundary(bag_clf, X, y)
+plt.title("Decision Trees with Bagging", fontsize=14)
+plt.show()
@@ -255,7 +252,6 @@ voting_clf.fit(X_train, y_train)
36
37
38
- 39
»
diff --git a/doc/pub/DecisionTrees/html/._DecisionTrees-bs037.html b/doc/pub/DecisionTrees/html/._DecisionTrees-bs037.html
index fd4b8a4d6..397c77e98 100644
--- a/doc/pub/DecisionTrees/html/._DecisionTrees-bs037.html
+++ b/doc/pub/DecisionTrees/html/._DecisionTrees-bs037.html
@@ -47,52 +47,47 @@ Automatically generated HTML file from DocOnce source
2,
None,
'___sec1'),
- ('A typical Decision Tree with its pertinent Jargon, Regeression '
- 'Problem',
- 2,
- None,
- '___sec2'),
- ('General Features', 2, None, '___sec3'),
- ('How do we set it up?', 2, None, '___sec4'),
- ('Decision trees and Regression', 2, None, '___sec5'),
- ('Building a tree, regression', 2, None, '___sec6'),
+ ('General Features', 2, None, '___sec2'),
+ ('How do we set it up?', 2, None, '___sec3'),
+ ('Decision trees and Regression', 2, None, '___sec4'),
+ ('Building a tree, regression', 2, None, '___sec5'),
('A top-down approach, recursive binary splitting',
2,
None,
- '___sec7'),
- ('Making a tree', 2, None, '___sec8'),
- ('Pruning the tree', 2, None, '___sec9'),
- ('Cost complexity pruning', 2, None, '___sec10'),
- ('Schematic Regression Procedure', 2, None, '___sec11'),
- ('A Classification Tree', 2, None, '___sec12'),
- ('Growing a classification tree', 2, None, '___sec13'),
- ('Classification tree, how to split nodes', 2, None, '___sec14'),
- ('Visualizing the Tree, Classification', 2, None, '___sec15'),
- ('Visualizing the Tree, The Moons', 2, None, '___sec16'),
- ('Computing the Gini index', 2, None, '___sec17'),
- ('Simple Python Code to read in Data', 2, None, '___sec18'),
- ('Computing the Gini Factor', 2, None, '___sec19'),
- ('Entropy and the ID3 algorithm', 2, None, '___sec20'),
- ('Implementing the ID3 Algorithm', 2, None, '___sec21'),
+ '___sec6'),
+ ('Making a tree', 2, None, '___sec7'),
+ ('Pruning the tree', 2, None, '___sec8'),
+ ('Cost complexity pruning', 2, None, '___sec9'),
+ ('Schematic Regression Procedure', 2, None, '___sec10'),
+ ('A Classification Tree', 2, None, '___sec11'),
+ ('Growing a classification tree', 2, None, '___sec12'),
+ ('Classification tree, how to split nodes', 2, None, '___sec13'),
+ ('Visualizing the Tree, Classification', 2, None, '___sec14'),
+ ('Visualizing the Tree, The Moons', 2, None, '___sec15'),
+ ('Computing the Gini index', 2, None, '___sec16'),
+ ('Simple Python Code to read in Data', 2, None, '___sec17'),
+ ('Computing the Gini Factor', 2, None, '___sec18'),
+ ('Entropy and the ID3 algorithm', 2, None, '___sec19'),
+ ('Implementing the ID3 Algorithm', 2, None, '___sec20'),
('Cancer Data again now with Decision Trees and other Methods',
2,
None,
- '___sec22'),
- ('Another example, the moons again', 2, None, '___sec23'),
- ('Playing around with regions', 2, None, '___sec24'),
- ('Regression trees', 2, None, '___sec25'),
- ('Final regressor code', 2, None, '___sec26'),
- ('Pros and cons of trees, pros', 2, None, '___sec27'),
- ('Disadvantages', 2, None, '___sec28'),
- ('Bagging', 2, None, '___sec29'),
- ('More bagging', 2, None, '___sec30'),
- ('Simple example, head or tail', 2, None, '___sec31'),
- ('Bagging Example', 2, None, '___sec32'),
- ('Random forests', 2, None, '___sec33'),
- ('A simple scikit-learn example', 2, None, '___sec34'),
- ('Please, not the moons again!', 2, None, '___sec35'),
- ('Bagging examples', 2, None, '___sec36'),
- ('Then random forests', 2, None, '___sec37')]}
+ '___sec21'),
+ ('Another example, the moons again', 2, None, '___sec22'),
+ ('Playing around with regions', 2, None, '___sec23'),
+ ('Regression trees', 2, None, '___sec24'),
+ ('Final regressor code', 2, None, '___sec25'),
+ ('Pros and cons of trees, pros', 2, None, '___sec26'),
+ ('Disadvantages', 2, None, '___sec27'),
+ ('Bagging', 2, None, '___sec28'),
+ ('More bagging', 2, None, '___sec29'),
+ ('Simple example, head or tail', 2, None, '___sec30'),
+ ('Bagging Example', 2, None, '___sec31'),
+ ('Random forests', 2, None, '___sec32'),
+ ('A simple scikit-learn example', 2, None, '___sec33'),
+ ('Please, not the moons again!', 2, None, '___sec34'),
+ ('Bagging examples', 2, None, '___sec35'),
+ ('Then random forests', 2, None, '___sec36')]}
end of tocinfo -->
@@ -132,42 +127,41 @@ MathJax.Hub.Config({
@@ -183,65 +177,27 @@ MathJax.Hub.Config({
-Bagging examples
-
+Then random forests
-
from sklearn.ensemble import BaggingClassifier
-from sklearn.tree import DecisionTreeClassifier
+bag_clf = BaggingClassifier(
+ DecisionTreeClassifier(splitter="random", max_leaf_nodes=16, random_state=42),
+ n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1, random_state=42)
+
+
-bag_clf = BaggingClassifier(
- DecisionTreeClassifier(random_state=42), n_estimators=500,
- max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)
-bag_clf.fit(X_train, y_train)
+
+
bag_clf.fit(X_train, y_train)
y_pred = bag_clf.predict(X_test)
+from sklearn.ensemble import RandomForestClassifier
+rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)
+rnd_clf.fit(X_train, y_train)
+y_pred_rf = rnd_clf.predict(X_test)
+np.sum(y_pred == y_pred_rf) / len(y_pred)
-
-
from sklearn.metrics import accuracy_score
-print(accuracy_score(y_test, y_pred))
-
-
-
-
-
tree_clf = DecisionTreeClassifier(random_state=42)
-tree_clf.fit(X_train, y_train)
-y_pred_tree = tree_clf.predict(X_test)
-print(accuracy_score(y_test, y_pred_tree))
-
-
-
-
-
from matplotlib.colors import ListedColormap
-
-def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):
- x1s = np.linspace(axes[0], axes[1], 100)
- x2s = np.linspace(axes[2], axes[3], 100)
- x1, x2 = np.meshgrid(x1s, x2s)
- X_new = np.c_[x1.ravel(), x2.ravel()]
- y_pred = clf.predict(X_new).reshape(x1.shape)
- custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
- plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
- if contour:
- custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
- plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
- plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha)
- plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha)
- plt.axis(axes)
- plt.xlabel(r"$x_1$", fontsize=18)
- plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
-plt.figure(figsize=(11,4))
-plt.subplot(121)
-plot_decision_boundary(tree_clf, X, y)
-plt.title("Decision Tree", fontsize=14)
-plt.subplot(122)
-plot_decision_boundary(bag_clf, X, y)
-plt.title("Decision Trees with Bagging", fontsize=14)
-plt.show()
-
-
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-bs.html b/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
index 4591c0dc0..ab79ad196 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-bs.html
@@ -47,52 +47,47 @@ Automatically generated HTML file from DocOnce source
2,
None,
'___sec1'),
- ('A typical Decision Tree with its pertinent Jargon, Regeression '
- 'Problem',
- 2,
- None,
- '___sec2'),
- ('General Features', 2, None, '___sec3'),
- ('How do we set it up?', 2, None, '___sec4'),
- ('Decision trees and Regression', 2, None, '___sec5'),
- ('Building a tree, regression', 2, None, '___sec6'),
+ ('General Features', 2, None, '___sec2'),
+ ('How do we set it up?', 2, None, '___sec3'),
+ ('Decision trees and Regression', 2, None, '___sec4'),
+ ('Building a tree, regression', 2, None, '___sec5'),
('A top-down approach, recursive binary splitting',
2,
None,
- '___sec7'),
- ('Making a tree', 2, None, '___sec8'),
- ('Pruning the tree', 2, None, '___sec9'),
- ('Cost complexity pruning', 2, None, '___sec10'),
- ('Schematic Regression Procedure', 2, None, '___sec11'),
- ('A Classification Tree', 2, None, '___sec12'),
- ('Growing a classification tree', 2, None, '___sec13'),
- ('Classification tree, how to split nodes', 2, None, '___sec14'),
- ('Visualizing the Tree, Classification', 2, None, '___sec15'),
- ('Visualizing the Tree, The Moons', 2, None, '___sec16'),
- ('Computing the Gini index', 2, None, '___sec17'),
- ('Simple Python Code to read in Data', 2, None, '___sec18'),
- ('Computing the Gini Factor', 2, None, '___sec19'),
- ('Entropy and the ID3 algorithm', 2, None, '___sec20'),
- ('Implementing the ID3 Algorithm', 2, None, '___sec21'),
+ '___sec6'),
+ ('Making a tree', 2, None, '___sec7'),
+ ('Pruning the tree', 2, None, '___sec8'),
+ ('Cost complexity pruning', 2, None, '___sec9'),
+ ('Schematic Regression Procedure', 2, None, '___sec10'),
+ ('A Classification Tree', 2, None, '___sec11'),
+ ('Growing a classification tree', 2, None, '___sec12'),
+ ('Classification tree, how to split nodes', 2, None, '___sec13'),
+ ('Visualizing the Tree, Classification', 2, None, '___sec14'),
+ ('Visualizing the Tree, The Moons', 2, None, '___sec15'),
+ ('Computing the Gini index', 2, None, '___sec16'),
+ ('Simple Python Code to read in Data', 2, None, '___sec17'),
+ ('Computing the Gini Factor', 2, None, '___sec18'),
+ ('Entropy and the ID3 algorithm', 2, None, '___sec19'),
+ ('Implementing the ID3 Algorithm', 2, None, '___sec20'),
('Cancer Data again now with Decision Trees and other Methods',
2,
None,
- '___sec22'),
- ('Another example, the moons again', 2, None, '___sec23'),
- ('Playing around with regions', 2, None, '___sec24'),
- ('Regression trees', 2, None, '___sec25'),
- ('Final regressor code', 2, None, '___sec26'),
- ('Pros and cons of trees, pros', 2, None, '___sec27'),
- ('Disadvantages', 2, None, '___sec28'),
- ('Bagging', 2, None, '___sec29'),
- ('More bagging', 2, None, '___sec30'),
- ('Simple example, head or tail', 2, None, '___sec31'),
- ('Bagging Example', 2, None, '___sec32'),
- ('Random forests', 2, None, '___sec33'),
- ('A simple scikit-learn example', 2, None, '___sec34'),
- ('Please, not the moons again!', 2, None, '___sec35'),
- ('Bagging examples', 2, None, '___sec36'),
- ('Then random forests', 2, None, '___sec37')]}
+ '___sec21'),
+ ('Another example, the moons again', 2, None, '___sec22'),
+ ('Playing around with regions', 2, None, '___sec23'),
+ ('Regression trees', 2, None, '___sec24'),
+ ('Final regressor code', 2, None, '___sec25'),
+ ('Pros and cons of trees, pros', 2, None, '___sec26'),
+ ('Disadvantages', 2, None, '___sec27'),
+ ('Bagging', 2, None, '___sec28'),
+ ('More bagging', 2, None, '___sec29'),
+ ('Simple example, head or tail', 2, None, '___sec30'),
+ ('Bagging Example', 2, None, '___sec31'),
+ ('Random forests', 2, None, '___sec32'),
+ ('A simple scikit-learn example', 2, None, '___sec33'),
+ ('Please, not the moons again!', 2, None, '___sec34'),
+ ('Bagging examples', 2, None, '___sec35'),
+ ('Then random forests', 2, None, '___sec36')]}
end of tocinfo -->
@@ -132,42 +127,41 @@ MathJax.Hub.Config({
@@ -226,7 +220,7 @@ MathJax.Hub.Config({
9
10
...
- 39
+ 38
»
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html b/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
index 2b39f802b..aef2f82ea 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-reveal.html
@@ -194,22 +194,11 @@ given some assumptions, make predictions about the target feature value
A typical Decision Tree with its pertinent Jargon, Classification Problem
-
-
-In the figure here we present a decision tree obtained from a classification problem
-A typical Decision Tree with its pertinent Jargon, Regeression Problem
-
-
-In the figure we present a decision tree obtained from a simple regression problem
-
-
-
-
-General Features
+General Features
The overarching approach to decision trees is a top-down approach.
@@ -228,7 +217,7 @@ node.
-How do we set it up?
+How do we set it up?
In simplified terms, the process of training a decision tree and
@@ -247,7 +236,7 @@ Then we are essentially done!
-Decision trees and Regression
+Decision trees and Regression
@@ -344,7 +333,7 @@ plt.show()
-Building a tree, regression
+Building a tree, regression
There are mainly two steps
@@ -376,7 +365,7 @@ within box \( j \).
-A top-down approach, recursive binary splitting
+A top-down approach, recursive binary splitting
Unfortunately, it is computationally infeasible to consider every
@@ -395,7 +384,7 @@ better tree in some future step.
-Making a tree
+Making a tree
In order to implement the recursive binary splitting we start by selecting
@@ -452,7 +441,7 @@ region contains more than five observations.
-Pruning the tree
+Pruning the tree
The above procedure is rather straightforward, but leads often to
@@ -471,7 +460,7 @@ parameter \( \alpha \).
-Cost complexity pruning
+Cost complexity pruning
For each value of \( \alpha \) there corresponds a subtree \( T \in T_0 \) such that
$$
@@ -504,7 +493,7 @@ subtree corresponding to \( \alpha \).
-Schematic Regression Procedure
+Schematic Regression Procedure
@@ -529,7 +518,7 @@ subtree corresponding to \( \alpha \).
-A Classification Tree
+A Classification Tree
A classification tree is very similar to a regression tree, except
@@ -548,7 +537,7 @@ fall into that region.
-Growing a classification tree
+Growing a classification tree
The task of growing a
@@ -572,7 +561,7 @@ than is the classification error rate.
-Classification tree, how to split nodes
+Classification tree, how to split nodes
If our targets are the outcome of a classification process that takes
@@ -627,7 +616,7 @@ $$
-Visualizing the Tree, Classification
+Visualizing the Tree, Classification
@@ -669,7 +658,7 @@ os.system(cmd)
-Visualizing the Tree, The Moons
+Visualizing the Tree, The Moons
@@ -702,7 +691,7 @@ os.system(cmd)
-Computing the Gini index
+Computing the Gini index
The example we will look at is a classical one in many Machine
@@ -743,7 +732,7 @@ The table here summarizes the various attributes and
-Simple Python Code to read in Data
+Simple Python Code to read in Data
@@ -806,7 +795,7 @@ display(y)
-Computing the Gini Factor
+Computing the Gini Factor
The above functions (gini, entropy and misclassification error) are
@@ -885,7 +874,7 @@ split = get_split(dataset)
-Entropy and the ID3 algorithm
+Entropy and the ID3 algorithm
ID3, learns decision trees by constructing
@@ -922,216 +911,204 @@ attributes at each step while growing the tree.
-Implementing the ID3 Algorithm
+Implementing the ID3 Algorithm
-import re
-import math
-from collections import deque
-
-
-
-
-
-
+
+
import re
+import math
+from collections import deque
-
-class Node(object):
- def __init__(self):
- self.value = None
- self.next = None
- self.childs = None
+# x is examples in training set
+# y is set of targets
+# label is target attributes
+# Node is a class which has properties values, childs, and next
+# root is top node in the decision tree
-
-
-
-class DecisionTree(object):
- def __init__(self, sample, attributes, labels):
- self.sample = sample
- self.attributes = attributes
- self.labels = labels
- self.labelCodes = None
- self.labelCodesCount = None
- self.initLabelCodes()
- # print(self.labelCodes)
- self.root = None
- self.entropy = self.getEntropy([x for x in range(len(self.labels))])
+class Node(object):
+ def __init__(self):
+ self.value = None
+ self.next = None
+ self.childs = None
-
- def initLabelCodes(self):
- self.labelCodes = []
- self.labelCodesCount = []
- for l in self.labels:
- if l not in self.labelCodes:
- self.labelCodes.append(l)
- self.labelCodesCount.append(0)
- self.labelCodesCount[self.labelCodes.index(l)] += 1
+# Simple class of Decision Tree
+# Aimed for who want to learn Decision Tree, so it is not optimized
+class DecisionTree(object):
+ def __init__(self, sample, attributes, labels):
+ self.sample = sample
+ self.attributes = attributes
+ self.labels = labels
+ self.labelCodes = None
+ self.labelCodesCount = None
+ self.initLabelCodes()
+ # print(self.labelCodes)
+ self.root = None
+ self.entropy = self.getEntropy([x for x in range(len(self.labels))])
-
- def getLabelCodeId(self, sampleId):
- return self.labelCodes.index(self.labels[sampleId])
+ def initLabelCodes(self):
+ self.labelCodes = []
+ self.labelCodesCount = []
+ for l in self.labels:
+ if l not in self.labelCodes:
+ self.labelCodes.append(l)
+ self.labelCodesCount.append(0)
+ self.labelCodesCount[self.labelCodes.index(l)] += 1
-
- def getAttributeValues(self, sampleIds, attributeId):
+ def getLabelCodeId(self, sampleId):
+ return self.labelCodes.index(self.labels[sampleId])
+
+ def getAttributeValues(self, sampleIds, attributeId):
vals = []
- for sid in sampleIds:
- val = self.sample[sid][attributeId]
- if val not in vals:
+ for sid in sampleIds:
+ val = self.sample[sid][attributeId]
+ if val not in vals:
vals.append(val)
- # print(vals)
- return vals
+ # print(vals)
+ return vals
-
- def getEntropy(self, sampleIds):
- entropy = 0
- labelCount = [0] * len(self.labelCodes)
- for sid in sampleIds:
- labelCount[self.getLabelCodeId(sid)] += 1
- # print("-ge", labelCount)
- for lv in labelCount:
- # print(lv)
- if lv != 0:
- entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
- else:
- entropy += 0
- return entropy
+ def getEntropy(self, sampleIds):
+ entropy = 0
+ labelCount = [0] * len(self.labelCodes)
+ for sid in sampleIds:
+ labelCount[self.getLabelCodeId(sid)] += 1
+ # print("-ge", labelCount)
+ for lv in labelCount:
+ # print(lv)
+ if lv != 0:
+ entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
+ else:
+ entropy += 0
+ return entropy
-
- def getDominantLabel(self, sampleIds):
- labelCodesCount = [0] * len(self.labelCodes)
- for sid in sampleIds:
- labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
- return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
+ def getDominantLabel(self, sampleIds):
+ labelCodesCount = [0] * len(self.labelCodes)
+ for sid in sampleIds:
+ labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
+ return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
-
- def getInformationGain(self, sampleIds, attributeId):
- gain = self.getEntropy(sampleIds)
+ def getInformationGain(self, sampleIds, attributeId):
+ gain = self.getEntropy(sampleIds)
attributeVals = []
attributeValsCount = []
attributeValsIds = []
- for sid in sampleIds:
- val = self.sample[sid][attributeId]
- if val not in attributeVals:
+ for sid in sampleIds:
+ val = self.sample[sid][attributeId]
+ if val not in attributeVals:
attributeVals.append(val)
- attributeValsCount.append(0)
+ attributeValsCount.append(0)
attributeValsIds.append([])
vid = attributeVals.index(val)
- attributeValsCount[vid] += 1
+ attributeValsCount[vid] += 1
attributeValsIds[vid].append(sid)
- # print("-gig", self.attributes[attributeId])
- for vc, vids in zip(attributeValsCount, attributeValsIds):
- # print("-gig", vids)
- gain -= vc/len(sampleIds) * self.getEntropy(vids)
- return gain
+ # print("-gig", self.attributes[attributeId])
+ for vc, vids in zip(attributeValsCount, attributeValsIds):
+ # print("-gig", vids)
+ gain -= vc/len(sampleIds) * self.getEntropy(vids)
+ return gain
-
- def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
- attributesEntropy = [0] * len(attributeIds)
- for i, attId in zip(range(len(attributeIds)), attributeIds):
- attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
- maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
- return self.attributes[maxId], maxId
+ def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
+ attributesEntropy = [0] * len(attributeIds)
+ for i, attId in zip(range(len(attributeIds)), attributeIds):
+ attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
+ maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
+ return self.attributes[maxId], maxId
-
- def isSingleLabeled(self, sampleIds):
- label = self.labels[sampleIds[0]]
- for sid in sampleIds:
- if self.labels[sid] != label:
- return False
- return True
+ def isSingleLabeled(self, sampleIds):
+ label = self.labels[sampleIds[0]]
+ for sid in sampleIds:
+ if self.labels[sid] != label:
+ return False
+ return True
-
- def getLabel(self, sampleId):
- return self.labels[sampleId]
+ def getLabel(self, sampleId):
+ return self.labels[sampleId]
-
- def id3(self):
- sampleIds = [x for x in range(len(self.sample))]
- attributeIds = [x for x in range(len(self.attributes))]
- self.root = self.id3Recv(sampleIds, attributeIds, self.root)
+ def id3(self):
+ sampleIds = [x for x in range(len(self.sample))]
+ attributeIds = [x for x in range(len(self.attributes))]
+ self.root = self.id3Recv(sampleIds, attributeIds, self.root)
-
- def id3Recv(self, sampleIds, attributeIds, root):
- root = Node() # Initialize current root
- if self.isSingleLabeled(sampleIds):
- root.value = self.labels[sampleIds[0]]
- return root
- # print(attributeIds)
- if len(attributeIds) == 0:
- root.value = self.getDominantLabel(sampleIds)
- return root
- bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
+ def id3Recv(self, sampleIds, attributeIds, root):
+ root = Node() # Initialize current root
+ if self.isSingleLabeled(sampleIds):
+ root.value = self.labels[sampleIds[0]]
+ return root
+ # print(attributeIds)
+ if len(attributeIds) == 0:
+ root.value = self.getDominantLabel(sampleIds)
+ return root
+ bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
sampleIds, attributeIds)
- # print(bestAttrName)
+ # print(bestAttrName)
root.value = bestAttrName
- root.childs = [] # Create list of children
- for value in self.getAttributeValues(sampleIds, bestAttrId):
- # print(value)
+ root.childs = [] # Create list of children
+ for value in self.getAttributeValues(sampleIds, bestAttrId):
+ # print(value)
child = Node()
child.value = value
- root.childs.append(child) # Append new child node to current
- # root
+ root.childs.append(child) # Append new child node to current
+ # root
childSampleIds = []
- for sid in sampleIds:
- if self.sample[sid][bestAttrId] == value:
+ for sid in sampleIds:
+ if self.sample[sid][bestAttrId] == value:
childSampleIds.append(sid)
- if len(childSampleIds) == 0:
- child.next = self.getDominantLabel(sampleIds)
- else:
- # print(bestAttrName, bestAttrId)
- # print(attributeIds)
- if len(attributeIds) > 0 and bestAttrId in attributeIds:
+ if len(childSampleIds) == 0:
+ child.next = self.getDominantLabel(sampleIds)
+ else:
+ # print(bestAttrName, bestAttrId)
+ # print(attributeIds)
+ if len(attributeIds) > 0 and bestAttrId in attributeIds:
toRemove = attributeIds.index(bestAttrId)
attributeIds.pop(toRemove)
- child.next = self.id3Recv(
+ child.next = self.id3Recv(
childSampleIds, attributeIds, child.next)
- return root
+ return root
-
- def printTree(self):
- if self.root:
+ def printTree(self):
+ if self.root:
roots = deque()
- roots.append(self.root)
- while len(roots) > 0:
+ roots.append(self.root)
+ while len(roots) > 0:
root = roots.popleft()
- print(root.value)
- if root.childs:
- for child in root.childs:
- print('({})'.format(child.value))
+ print(root.value)
+ if root.childs:
+ for child in root.childs:
+ print('({})'.format(child.value))
roots.append(child.next)
- elif root.next:
- print(root.next)
+ elif root.next:
+ print(root.next)
-
-def test():
- f = open('DataFiles/rideclass.csv')
- attributes = f.readline().split(',')
- attributes = attributes[1:len(attributes)-1]
- print(attributes)
+
+def test():
+ f = open('DataFiles/rideclass.csv')
+ attributes = f.readline().split(',')
+ attributes = attributes[1:len(attributes)-1]
+ print(attributes)
sample = f.readlines()
f.close()
- for i in range(len(sample)):
- sample[i] = re.sub('\d+,', '', sample[i])
- sample[i] = sample[i].strip().split(',')
+ for i in range(len(sample)):
+ sample[i] = re.sub('\d+,', '', sample[i])
+ sample[i] = sample[i].strip().split(',')
labels = []
- for s in sample:
+ for s in sample:
labels.append(s.pop())
- # print(sample)
- # print(labels)
+ # print(sample)
+ # print(labels)
decisionTree = DecisionTree(sample, attributes, labels)
- print("System entropy {}".format(decisionTree.entropy))
+ print("System entropy {}".format(decisionTree.entropy))
decisionTree.id3()
decisionTree.printTree()
-
-if __name__ == '__main__':
+
+if __name__ == '__main__':
test()
+
-Cancer Data again now with Decision Trees and other Methods
+Cancer Data again now with Decision Trees and other Methods
@@ -1181,7 +1158,7 @@ deep_tree_clf.fit(X_train_scaled, y_train)
-Another example, the moons again
+Another example, the moons again
@@ -1254,7 +1231,7 @@ plt.show()
-Playing around with regions
+Playing around with regions
@@ -1283,7 +1260,7 @@ plt.show()
-Regression trees
+Regression trees
@@ -1306,7 +1283,7 @@ tree_reg.fit(X, y)
-Final regressor code
+Final regressor code
@@ -1385,7 +1362,7 @@ plt.show()
-Pros and cons of trees, pros
+Pros and cons of trees, pros
- White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines)
@@ -1400,7 +1377,7 @@ plt.show()
-Disadvantages
+Disadvantages
- Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches
@@ -1418,7 +1395,7 @@ However, by aggregating many decision trees, using methods like bagging, random
-Bagging
+Bagging
The plain decision trees suffer from high
@@ -1437,7 +1414,7 @@ learning method.
-More bagging
+More bagging
Bagging typically results in improved accuracy
@@ -1466,7 +1443,7 @@ predictor, averaged over all \( B \) trees.
-Simple example, head or tail
+Simple example, head or tail
@@ -1487,7 +1464,7 @@ plt.show()
-Bagging Example
+Bagging Example
@@ -1539,7 +1516,7 @@ voting_clf.fit(X_train, y_train)
-Random forests
+Random forests
Random forests provide an improvement over bagged trees by way of a
@@ -1585,7 +1562,7 @@ this setting.
-A simple scikit-learn example
+A simple scikit-learn example
@@ -1604,7 +1581,7 @@ accuracy = cross_validate(Random_Forest_model,X,Y,cv=Please, not the moons again!
+Please, not the moons again!
@@ -1663,7 +1640,7 @@ voting_clf.fit(X_train, y_train)
-Bagging examples
+Bagging examples
@@ -1725,7 +1702,7 @@ plt.show()
-Then random forests
+Then random forests
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html b/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html
index b75715681..94d27f261 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees-solarized.html
@@ -67,52 +67,47 @@ div { text-align: justify; text-justify: inter-word; }
2,
None,
'___sec1'),
- ('A typical Decision Tree with its pertinent Jargon, Regeression '
- 'Problem',
- 2,
- None,
- '___sec2'),
- ('General Features', 2, None, '___sec3'),
- ('How do we set it up?', 2, None, '___sec4'),
- ('Decision trees and Regression', 2, None, '___sec5'),
- ('Building a tree, regression', 2, None, '___sec6'),
+ ('General Features', 2, None, '___sec2'),
+ ('How do we set it up?', 2, None, '___sec3'),
+ ('Decision trees and Regression', 2, None, '___sec4'),
+ ('Building a tree, regression', 2, None, '___sec5'),
('A top-down approach, recursive binary splitting',
2,
None,
- '___sec7'),
- ('Making a tree', 2, None, '___sec8'),
- ('Pruning the tree', 2, None, '___sec9'),
- ('Cost complexity pruning', 2, None, '___sec10'),
- ('Schematic Regression Procedure', 2, None, '___sec11'),
- ('A Classification Tree', 2, None, '___sec12'),
- ('Growing a classification tree', 2, None, '___sec13'),
- ('Classification tree, how to split nodes', 2, None, '___sec14'),
- ('Visualizing the Tree, Classification', 2, None, '___sec15'),
- ('Visualizing the Tree, The Moons', 2, None, '___sec16'),
- ('Computing the Gini index', 2, None, '___sec17'),
- ('Simple Python Code to read in Data', 2, None, '___sec18'),
- ('Computing the Gini Factor', 2, None, '___sec19'),
- ('Entropy and the ID3 algorithm', 2, None, '___sec20'),
- ('Implementing the ID3 Algorithm', 2, None, '___sec21'),
+ '___sec6'),
+ ('Making a tree', 2, None, '___sec7'),
+ ('Pruning the tree', 2, None, '___sec8'),
+ ('Cost complexity pruning', 2, None, '___sec9'),
+ ('Schematic Regression Procedure', 2, None, '___sec10'),
+ ('A Classification Tree', 2, None, '___sec11'),
+ ('Growing a classification tree', 2, None, '___sec12'),
+ ('Classification tree, how to split nodes', 2, None, '___sec13'),
+ ('Visualizing the Tree, Classification', 2, None, '___sec14'),
+ ('Visualizing the Tree, The Moons', 2, None, '___sec15'),
+ ('Computing the Gini index', 2, None, '___sec16'),
+ ('Simple Python Code to read in Data', 2, None, '___sec17'),
+ ('Computing the Gini Factor', 2, None, '___sec18'),
+ ('Entropy and the ID3 algorithm', 2, None, '___sec19'),
+ ('Implementing the ID3 Algorithm', 2, None, '___sec20'),
('Cancer Data again now with Decision Trees and other Methods',
2,
None,
- '___sec22'),
- ('Another example, the moons again', 2, None, '___sec23'),
- ('Playing around with regions', 2, None, '___sec24'),
- ('Regression trees', 2, None, '___sec25'),
- ('Final regressor code', 2, None, '___sec26'),
- ('Pros and cons of trees, pros', 2, None, '___sec27'),
- ('Disadvantages', 2, None, '___sec28'),
- ('Bagging', 2, None, '___sec29'),
- ('More bagging', 2, None, '___sec30'),
- ('Simple example, head or tail', 2, None, '___sec31'),
- ('Bagging Example', 2, None, '___sec32'),
- ('Random forests', 2, None, '___sec33'),
- ('A simple scikit-learn example', 2, None, '___sec34'),
- ('Please, not the moons again!', 2, None, '___sec35'),
- ('Bagging examples', 2, None, '___sec36'),
- ('Then random forests', 2, None, '___sec37')]}
+ '___sec21'),
+ ('Another example, the moons again', 2, None, '___sec22'),
+ ('Playing around with regions', 2, None, '___sec23'),
+ ('Regression trees', 2, None, '___sec24'),
+ ('Final regressor code', 2, None, '___sec25'),
+ ('Pros and cons of trees, pros', 2, None, '___sec26'),
+ ('Disadvantages', 2, None, '___sec27'),
+ ('Bagging', 2, None, '___sec28'),
+ ('More bagging', 2, None, '___sec29'),
+ ('Simple example, head or tail', 2, None, '___sec30'),
+ ('Bagging Example', 2, None, '___sec31'),
+ ('Random forests', 2, None, '___sec32'),
+ ('A simple scikit-learn example', 2, None, '___sec33'),
+ ('Please, not the moons again!', 2, None, '___sec34'),
+ ('Bagging examples', 2, None, '___sec35'),
+ ('Then random forests', 2, None, '___sec36')]}
end of tocinfo -->
@@ -195,21 +190,10 @@ given some assumptions, make predictions about the target feature value
A typical Decision Tree with its pertinent Jargon, Classification Problem
-
-In the figure here we present a decision tree obtained from a classification problem
-
-
A typical Decision Tree with its pertinent Jargon, Regeression Problem
-
-
-In the figure we present a decision tree obtained from a simple regression problem
-
-
-
-
-
General Features
+General Features
The overarching approach to decision trees is a top-down approach.
@@ -227,7 +211,7 @@ node.
-
How do we set it up?
+How do we set it up?
In simplified terms, the process of training a decision tree and
@@ -245,7 +229,7 @@ Then we are essentially done!
-
Decision trees and Regression
+Decision trees and Regression
@@ -341,7 +325,7 @@ plt.show()
-
Building a tree, regression
+Building a tree, regression
There are mainly two steps
@@ -369,7 +353,7 @@ within box \( j \).
-
A top-down approach, recursive binary splitting
+A top-down approach, recursive binary splitting
Unfortunately, it is computationally infeasible to consider every
@@ -388,7 +372,7 @@ better tree in some future step.
-
Making a tree
+Making a tree
In order to implement the recursive binary splitting we start by selecting
@@ -439,7 +423,7 @@ region contains more than five observations.
-
Pruning the tree
+Pruning the tree
The above procedure is rather straightforward, but leads often to
@@ -458,7 +442,7 @@ parameter \( \alpha \).
-
Cost complexity pruning
+Cost complexity pruning
For each value of \( \alpha \) there corresponds a subtree \( T \in T_0 \) such that
$$
\sum_{m=1}^{\overline{T}}\sum_{i:x_i\in R_m}(y_i-\overline{y}_{R_m})^2+\alpha\overline{T},
@@ -489,7 +473,7 @@ subtree corresponding to \( \alpha \).
-
Schematic Regression Procedure
+Schematic Regression Procedure
@@ -515,7 +499,7 @@ subtree corresponding to \( \alpha \).
-
A Classification Tree
+A Classification Tree
A classification tree is very similar to a regression tree, except
@@ -534,7 +518,7 @@ fall into that region.
-
Growing a classification tree
+Growing a classification tree
The task of growing a
@@ -558,7 +542,7 @@ than is the classification error rate.
-
Classification tree, how to split nodes
+Classification tree, how to split nodes
If our targets are the outcome of a classification process that takes
@@ -608,7 +592,7 @@ $$
-
Visualizing the Tree, Classification
+Visualizing the Tree, Classification
@@ -649,7 +633,7 @@ os.system(cmd)
-
Visualizing the Tree, The Moons
+Visualizing the Tree, The Moons
@@ -681,7 +665,7 @@ os.system(cmd)
-
Computing the Gini index
+Computing the Gini index
The example we will look at is a classical one in many Machine
@@ -721,7 +705,7 @@ The table here summarizes the various attributes and
-
Simple Python Code to read in Data
+Simple Python Code to read in Data
@@ -783,7 +767,7 @@ display(y)
-
Computing the Gini Factor
+Computing the Gini Factor
The above functions (gini, entropy and misclassification error) are
@@ -861,7 +845,7 @@ split = get_split(dataset)
-
Entropy and the ID3 algorithm
+Entropy and the ID3 algorithm
ID3, learns decision trees by constructing
@@ -897,216 +881,203 @@ attributes at each step while growing the tree.
-
Implementing the ID3 Algorithm
+Implementing the ID3 Algorithm
-import re
-import math
-from collections import deque
-
-
-
-
-
-
+
+
import re
+import math
+from collections import deque
-
-class Node(object):
- def __init__(self):
- self.value = None
- self.next = None
- self.childs = None
+# x is examples in training set
+# y is set of targets
+# label is target attributes
+# Node is a class which has properties values, childs, and next
+# root is top node in the decision tree
-
-
-
-class DecisionTree(object):
- def __init__(self, sample, attributes, labels):
- self.sample = sample
- self.attributes = attributes
- self.labels = labels
- self.labelCodes = None
- self.labelCodesCount = None
- self.initLabelCodes()
- # print(self.labelCodes)
- self.root = None
- self.entropy = self.getEntropy([x for x in range(len(self.labels))])
+class Node(object):
+ def __init__(self):
+ self.value = None
+ self.next = None
+ self.childs = None
-
- def initLabelCodes(self):
- self.labelCodes = []
- self.labelCodesCount = []
- for l in self.labels:
- if l not in self.labelCodes:
- self.labelCodes.append(l)
- self.labelCodesCount.append(0)
- self.labelCodesCount[self.labelCodes.index(l)] += 1
+# Simple class of Decision Tree
+# Aimed for who want to learn Decision Tree, so it is not optimized
+class DecisionTree(object):
+ def __init__(self, sample, attributes, labels):
+ self.sample = sample
+ self.attributes = attributes
+ self.labels = labels
+ self.labelCodes = None
+ self.labelCodesCount = None
+ self.initLabelCodes()
+ # print(self.labelCodes)
+ self.root = None
+ self.entropy = self.getEntropy([x for x in range(len(self.labels))])
-
- def getLabelCodeId(self, sampleId):
- return self.labelCodes.index(self.labels[sampleId])
+ def initLabelCodes(self):
+ self.labelCodes = []
+ self.labelCodesCount = []
+ for l in self.labels:
+ if l not in self.labelCodes:
+ self.labelCodes.append(l)
+ self.labelCodesCount.append(0)
+ self.labelCodesCount[self.labelCodes.index(l)] += 1
-
- def getAttributeValues(self, sampleIds, attributeId):
+ def getLabelCodeId(self, sampleId):
+ return self.labelCodes.index(self.labels[sampleId])
+
+ def getAttributeValues(self, sampleIds, attributeId):
vals = []
- for sid in sampleIds:
- val = self.sample[sid][attributeId]
- if val not in vals:
+ for sid in sampleIds:
+ val = self.sample[sid][attributeId]
+ if val not in vals:
vals.append(val)
- # print(vals)
- return vals
+ # print(vals)
+ return vals
-
- def getEntropy(self, sampleIds):
- entropy = 0
- labelCount = [0] * len(self.labelCodes)
- for sid in sampleIds:
- labelCount[self.getLabelCodeId(sid)] += 1
- # print("-ge", labelCount)
- for lv in labelCount:
- # print(lv)
- if lv != 0:
- entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
- else:
- entropy += 0
- return entropy
+ def getEntropy(self, sampleIds):
+ entropy = 0
+ labelCount = [0] * len(self.labelCodes)
+ for sid in sampleIds:
+ labelCount[self.getLabelCodeId(sid)] += 1
+ # print("-ge", labelCount)
+ for lv in labelCount:
+ # print(lv)
+ if lv != 0:
+ entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
+ else:
+ entropy += 0
+ return entropy
-
- def getDominantLabel(self, sampleIds):
- labelCodesCount = [0] * len(self.labelCodes)
- for sid in sampleIds:
- labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
- return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
+ def getDominantLabel(self, sampleIds):
+ labelCodesCount = [0] * len(self.labelCodes)
+ for sid in sampleIds:
+ labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
+ return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
-
- def getInformationGain(self, sampleIds, attributeId):
- gain = self.getEntropy(sampleIds)
+ def getInformationGain(self, sampleIds, attributeId):
+ gain = self.getEntropy(sampleIds)
attributeVals = []
attributeValsCount = []
attributeValsIds = []
- for sid in sampleIds:
- val = self.sample[sid][attributeId]
- if val not in attributeVals:
+ for sid in sampleIds:
+ val = self.sample[sid][attributeId]
+ if val not in attributeVals:
attributeVals.append(val)
- attributeValsCount.append(0)
+ attributeValsCount.append(0)
attributeValsIds.append([])
vid = attributeVals.index(val)
- attributeValsCount[vid] += 1
+ attributeValsCount[vid] += 1
attributeValsIds[vid].append(sid)
- # print("-gig", self.attributes[attributeId])
- for vc, vids in zip(attributeValsCount, attributeValsIds):
- # print("-gig", vids)
- gain -= vc/len(sampleIds) * self.getEntropy(vids)
- return gain
+ # print("-gig", self.attributes[attributeId])
+ for vc, vids in zip(attributeValsCount, attributeValsIds):
+ # print("-gig", vids)
+ gain -= vc/len(sampleIds) * self.getEntropy(vids)
+ return gain
-
- def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
- attributesEntropy = [0] * len(attributeIds)
- for i, attId in zip(range(len(attributeIds)), attributeIds):
- attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
- maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
- return self.attributes[maxId], maxId
+ def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
+ attributesEntropy = [0] * len(attributeIds)
+ for i, attId in zip(range(len(attributeIds)), attributeIds):
+ attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
+ maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
+ return self.attributes[maxId], maxId
-
- def isSingleLabeled(self, sampleIds):
- label = self.labels[sampleIds[0]]
- for sid in sampleIds:
- if self.labels[sid] != label:
- return False
- return True
+ def isSingleLabeled(self, sampleIds):
+ label = self.labels[sampleIds[0]]
+ for sid in sampleIds:
+ if self.labels[sid] != label:
+ return False
+ return True
-
- def getLabel(self, sampleId):
- return self.labels[sampleId]
+ def getLabel(self, sampleId):
+ return self.labels[sampleId]
-
- def id3(self):
- sampleIds = [x for x in range(len(self.sample))]
- attributeIds = [x for x in range(len(self.attributes))]
- self.root = self.id3Recv(sampleIds, attributeIds, self.root)
+ def id3(self):
+ sampleIds = [x for x in range(len(self.sample))]
+ attributeIds = [x for x in range(len(self.attributes))]
+ self.root = self.id3Recv(sampleIds, attributeIds, self.root)
-
- def id3Recv(self, sampleIds, attributeIds, root):
- root = Node() # Initialize current root
- if self.isSingleLabeled(sampleIds):
- root.value = self.labels[sampleIds[0]]
- return root
- # print(attributeIds)
- if len(attributeIds) == 0:
- root.value = self.getDominantLabel(sampleIds)
- return root
- bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
+ def id3Recv(self, sampleIds, attributeIds, root):
+ root = Node() # Initialize current root
+ if self.isSingleLabeled(sampleIds):
+ root.value = self.labels[sampleIds[0]]
+ return root
+ # print(attributeIds)
+ if len(attributeIds) == 0:
+ root.value = self.getDominantLabel(sampleIds)
+ return root
+ bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
sampleIds, attributeIds)
- # print(bestAttrName)
+ # print(bestAttrName)
root.value = bestAttrName
- root.childs = [] # Create list of children
- for value in self.getAttributeValues(sampleIds, bestAttrId):
- # print(value)
+ root.childs = [] # Create list of children
+ for value in self.getAttributeValues(sampleIds, bestAttrId):
+ # print(value)
child = Node()
child.value = value
- root.childs.append(child) # Append new child node to current
- # root
+ root.childs.append(child) # Append new child node to current
+ # root
childSampleIds = []
- for sid in sampleIds:
- if self.sample[sid][bestAttrId] == value:
+ for sid in sampleIds:
+ if self.sample[sid][bestAttrId] == value:
childSampleIds.append(sid)
- if len(childSampleIds) == 0:
- child.next = self.getDominantLabel(sampleIds)
- else:
- # print(bestAttrName, bestAttrId)
- # print(attributeIds)
- if len(attributeIds) > 0 and bestAttrId in attributeIds:
+ if len(childSampleIds) == 0:
+ child.next = self.getDominantLabel(sampleIds)
+ else:
+ # print(bestAttrName, bestAttrId)
+ # print(attributeIds)
+ if len(attributeIds) > 0 and bestAttrId in attributeIds:
toRemove = attributeIds.index(bestAttrId)
attributeIds.pop(toRemove)
- child.next = self.id3Recv(
+ child.next = self.id3Recv(
childSampleIds, attributeIds, child.next)
- return root
+ return root
-
- def printTree(self):
- if self.root:
+ def printTree(self):
+ if self.root:
roots = deque()
- roots.append(self.root)
- while len(roots) > 0:
+ roots.append(self.root)
+ while len(roots) > 0:
root = roots.popleft()
- print(root.value)
- if root.childs:
- for child in root.childs:
- print('({})'.format(child.value))
+ print(root.value)
+ if root.childs:
+ for child in root.childs:
+ print('({})'.format(child.value))
roots.append(child.next)
- elif root.next:
- print(root.next)
+ elif root.next:
+ print(root.next)
-
-def test():
- f = open('DataFiles/rideclass.csv')
- attributes = f.readline().split(',')
- attributes = attributes[1:len(attributes)-1]
- print(attributes)
+
+def test():
+ f = open('DataFiles/rideclass.csv')
+ attributes = f.readline().split(',')
+ attributes = attributes[1:len(attributes)-1]
+ print(attributes)
sample = f.readlines()
f.close()
- for i in range(len(sample)):
- sample[i] = re.sub('\d+,', '', sample[i])
- sample[i] = sample[i].strip().split(',')
+ for i in range(len(sample)):
+ sample[i] = re.sub('\d+,', '', sample[i])
+ sample[i] = sample[i].strip().split(',')
labels = []
- for s in sample:
+ for s in sample:
labels.append(s.pop())
- # print(sample)
- # print(labels)
+ # print(sample)
+ # print(labels)
decisionTree = DecisionTree(sample, attributes, labels)
- print("System entropy {}".format(decisionTree.entropy))
+ print("System entropy {}".format(decisionTree.entropy))
decisionTree.id3()
decisionTree.printTree()
-
-if __name__ == '__main__':
- test()
+if __name__ == '__main__':
+ test()
+
-
Cancer Data again now with Decision Trees and other Methods
+Cancer Data again now with Decision Trees and other Methods
@@ -1155,7 +1126,7 @@ deep_tree_clf.fit(X_train_scaled, y_train)
-
Another example, the moons again
+Another example, the moons again
@@ -1227,7 +1198,7 @@ plt.show()
-
Playing around with regions
+Playing around with regions
@@ -1255,7 +1226,7 @@ plt.show()
-
Regression trees
+Regression trees
@@ -1277,7 +1248,7 @@ tree_reg.fit(X, y)
-
Final regressor code
+Final regressor code
@@ -1355,7 +1326,7 @@ plt.show()
-
Pros and cons of trees, pros
+Pros and cons of trees, pros
- White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines)
@@ -1369,7 +1340,7 @@ plt.show()
-Disadvantages
+Disadvantages
- Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches
@@ -1386,7 +1357,7 @@ However, by aggregating many decision trees, using methods like bagging, random
-
Bagging
+Bagging
The plain decision trees suffer from high
@@ -1405,7 +1376,7 @@ learning method.
-
More bagging
+More bagging
Bagging typically results in improved accuracy
@@ -1434,7 +1405,7 @@ predictor, averaged over all \( B \) trees.
-
Simple example, head or tail
+Simple example, head or tail
@@ -1454,7 +1425,7 @@ plt.show()
-
Bagging Example
+Bagging Example
@@ -1505,7 +1476,7 @@ voting_clf.fit(X_train, y_train)
-
Random forests
+Random forests
Random forests provide an improvement over bagged trees by way of a
@@ -1549,7 +1520,7 @@ this setting.
-
A simple scikit-learn example
+A simple scikit-learn example
@@ -1567,7 +1538,7 @@ accuracy = cross_validate(Random_Forest_model,X,Y,cv=Please, not the moons again!
+Please, not the moons again!
@@ -1625,7 +1596,7 @@ voting_clf.fit(X_train, y_train)
-
Bagging examples
+Bagging examples
@@ -1686,7 +1657,7 @@ plt.show()
-
Then random forests
+Then random forests
diff --git a/doc/pub/DecisionTrees/html/DecisionTrees.html b/doc/pub/DecisionTrees/html/DecisionTrees.html
index 5885a1674..d41c8cb6d 100644
--- a/doc/pub/DecisionTrees/html/DecisionTrees.html
+++ b/doc/pub/DecisionTrees/html/DecisionTrees.html
@@ -72,52 +72,47 @@ div { text-align: justify; text-justify: inter-word; }
2,
None,
'___sec1'),
- ('A typical Decision Tree with its pertinent Jargon, Regeression '
- 'Problem',
- 2,
- None,
- '___sec2'),
- ('General Features', 2, None, '___sec3'),
- ('How do we set it up?', 2, None, '___sec4'),
- ('Decision trees and Regression', 2, None, '___sec5'),
- ('Building a tree, regression', 2, None, '___sec6'),
+ ('General Features', 2, None, '___sec2'),
+ ('How do we set it up?', 2, None, '___sec3'),
+ ('Decision trees and Regression', 2, None, '___sec4'),
+ ('Building a tree, regression', 2, None, '___sec5'),
('A top-down approach, recursive binary splitting',
2,
None,
- '___sec7'),
- ('Making a tree', 2, None, '___sec8'),
- ('Pruning the tree', 2, None, '___sec9'),
- ('Cost complexity pruning', 2, None, '___sec10'),
- ('Schematic Regression Procedure', 2, None, '___sec11'),
- ('A Classification Tree', 2, None, '___sec12'),
- ('Growing a classification tree', 2, None, '___sec13'),
- ('Classification tree, how to split nodes', 2, None, '___sec14'),
- ('Visualizing the Tree, Classification', 2, None, '___sec15'),
- ('Visualizing the Tree, The Moons', 2, None, '___sec16'),
- ('Computing the Gini index', 2, None, '___sec17'),
- ('Simple Python Code to read in Data', 2, None, '___sec18'),
- ('Computing the Gini Factor', 2, None, '___sec19'),
- ('Entropy and the ID3 algorithm', 2, None, '___sec20'),
- ('Implementing the ID3 Algorithm', 2, None, '___sec21'),
+ '___sec6'),
+ ('Making a tree', 2, None, '___sec7'),
+ ('Pruning the tree', 2, None, '___sec8'),
+ ('Cost complexity pruning', 2, None, '___sec9'),
+ ('Schematic Regression Procedure', 2, None, '___sec10'),
+ ('A Classification Tree', 2, None, '___sec11'),
+ ('Growing a classification tree', 2, None, '___sec12'),
+ ('Classification tree, how to split nodes', 2, None, '___sec13'),
+ ('Visualizing the Tree, Classification', 2, None, '___sec14'),
+ ('Visualizing the Tree, The Moons', 2, None, '___sec15'),
+ ('Computing the Gini index', 2, None, '___sec16'),
+ ('Simple Python Code to read in Data', 2, None, '___sec17'),
+ ('Computing the Gini Factor', 2, None, '___sec18'),
+ ('Entropy and the ID3 algorithm', 2, None, '___sec19'),
+ ('Implementing the ID3 Algorithm', 2, None, '___sec20'),
('Cancer Data again now with Decision Trees and other Methods',
2,
None,
- '___sec22'),
- ('Another example, the moons again', 2, None, '___sec23'),
- ('Playing around with regions', 2, None, '___sec24'),
- ('Regression trees', 2, None, '___sec25'),
- ('Final regressor code', 2, None, '___sec26'),
- ('Pros and cons of trees, pros', 2, None, '___sec27'),
- ('Disadvantages', 2, None, '___sec28'),
- ('Bagging', 2, None, '___sec29'),
- ('More bagging', 2, None, '___sec30'),
- ('Simple example, head or tail', 2, None, '___sec31'),
- ('Bagging Example', 2, None, '___sec32'),
- ('Random forests', 2, None, '___sec33'),
- ('A simple scikit-learn example', 2, None, '___sec34'),
- ('Please, not the moons again!', 2, None, '___sec35'),
- ('Bagging examples', 2, None, '___sec36'),
- ('Then random forests', 2, None, '___sec37')]}
+ '___sec21'),
+ ('Another example, the moons again', 2, None, '___sec22'),
+ ('Playing around with regions', 2, None, '___sec23'),
+ ('Regression trees', 2, None, '___sec24'),
+ ('Final regressor code', 2, None, '___sec25'),
+ ('Pros and cons of trees, pros', 2, None, '___sec26'),
+ ('Disadvantages', 2, None, '___sec27'),
+ ('Bagging', 2, None, '___sec28'),
+ ('More bagging', 2, None, '___sec29'),
+ ('Simple example, head or tail', 2, None, '___sec30'),
+ ('Bagging Example', 2, None, '___sec31'),
+ ('Random forests', 2, None, '___sec32'),
+ ('A simple scikit-learn example', 2, None, '___sec33'),
+ ('Please, not the moons again!', 2, None, '___sec34'),
+ ('Bagging examples', 2, None, '___sec35'),
+ ('Then random forests', 2, None, '___sec36')]}
end of tocinfo -->
@@ -200,21 +195,10 @@ given some assumptions, make predictions about the target feature value
A typical Decision Tree with its pertinent Jargon, Classification Problem
-
-In the figure here we present a decision tree obtained from a classification problem
-
-
A typical Decision Tree with its pertinent Jargon, Regeression Problem
-
-
-In the figure we present a decision tree obtained from a simple regression problem
-
-
-
-
-
General Features
+General Features
The overarching approach to decision trees is a top-down approach.
@@ -232,7 +216,7 @@ node.
-
How do we set it up?
+How do we set it up?
In simplified terms, the process of training a decision tree and
@@ -250,7 +234,7 @@ Then we are essentially done!
-
Decision trees and Regression
+Decision trees and Regression
@@ -346,7 +330,7 @@ plt.show()
-
Building a tree, regression
+Building a tree, regression
There are mainly two steps
@@ -374,7 +358,7 @@ within box \( j \).
-
A top-down approach, recursive binary splitting
+A top-down approach, recursive binary splitting
Unfortunately, it is computationally infeasible to consider every
@@ -393,7 +377,7 @@ better tree in some future step.
-
Making a tree
+Making a tree
In order to implement the recursive binary splitting we start by selecting
@@ -444,7 +428,7 @@ region contains more than five observations.
-
Pruning the tree
+Pruning the tree
The above procedure is rather straightforward, but leads often to
@@ -463,7 +447,7 @@ parameter \( \alpha \).
-
Cost complexity pruning
+Cost complexity pruning
For each value of \( \alpha \) there corresponds a subtree \( T \in T_0 \) such that
$$
\sum_{m=1}^{\overline{T}}\sum_{i:x_i\in R_m}(y_i-\overline{y}_{R_m})^2+\alpha\overline{T},
@@ -494,7 +478,7 @@ subtree corresponding to \( \alpha \).
-
Schematic Regression Procedure
+Schematic Regression Procedure
@@ -520,7 +504,7 @@ subtree corresponding to \( \alpha \).
-
A Classification Tree
+A Classification Tree
A classification tree is very similar to a regression tree, except
@@ -539,7 +523,7 @@ fall into that region.
-
Growing a classification tree
+Growing a classification tree
The task of growing a
@@ -563,7 +547,7 @@ than is the classification error rate.
-
Classification tree, how to split nodes
+Classification tree, how to split nodes
If our targets are the outcome of a classification process that takes
@@ -613,7 +597,7 @@ $$
-
Visualizing the Tree, Classification
+Visualizing the Tree, Classification
@@ -654,7 +638,7 @@ os.system(cmd)
-
Visualizing the Tree, The Moons
+Visualizing the Tree, The Moons
@@ -686,7 +670,7 @@ os.system(cmd)
-
Computing the Gini index
+Computing the Gini index
The example we will look at is a classical one in many Machine
@@ -726,7 +710,7 @@ The table here summarizes the various attributes and
-
Simple Python Code to read in Data
+Simple Python Code to read in Data
@@ -788,7 +772,7 @@ display(y)
-
Computing the Gini Factor
+Computing the Gini Factor
The above functions (gini, entropy and misclassification error) are
@@ -866,7 +850,7 @@ split = get_split(dataset)
-
Entropy and the ID3 algorithm
+Entropy and the ID3 algorithm
ID3, learns decision trees by constructing
@@ -902,216 +886,203 @@ attributes at each step while growing the tree.
-
Implementing the ID3 Algorithm
+Implementing the ID3 Algorithm
-import re
-import math
-from collections import deque
-
-
-
-
-
-
+
+
import re
+import math
+from collections import deque
-
-class Node(object):
- def __init__(self):
- self.value = None
- self.next = None
- self.childs = None
+# x is examples in training set
+# y is set of targets
+# label is target attributes
+# Node is a class which has properties values, childs, and next
+# root is top node in the decision tree
-
-
-
-class DecisionTree(object):
- def __init__(self, sample, attributes, labels):
- self.sample = sample
- self.attributes = attributes
- self.labels = labels
- self.labelCodes = None
- self.labelCodesCount = None
- self.initLabelCodes()
- # print(self.labelCodes)
- self.root = None
- self.entropy = self.getEntropy([x for x in range(len(self.labels))])
+class Node(object):
+ def __init__(self):
+ self.value = None
+ self.next = None
+ self.childs = None
-
- def initLabelCodes(self):
- self.labelCodes = []
- self.labelCodesCount = []
- for l in self.labels:
- if l not in self.labelCodes:
- self.labelCodes.append(l)
- self.labelCodesCount.append(0)
- self.labelCodesCount[self.labelCodes.index(l)] += 1
+# Simple class of Decision Tree
+# Aimed for who want to learn Decision Tree, so it is not optimized
+class DecisionTree(object):
+ def __init__(self, sample, attributes, labels):
+ self.sample = sample
+ self.attributes = attributes
+ self.labels = labels
+ self.labelCodes = None
+ self.labelCodesCount = None
+ self.initLabelCodes()
+ # print(self.labelCodes)
+ self.root = None
+ self.entropy = self.getEntropy([x for x in range(len(self.labels))])
-
- def getLabelCodeId(self, sampleId):
- return self.labelCodes.index(self.labels[sampleId])
+ def initLabelCodes(self):
+ self.labelCodes = []
+ self.labelCodesCount = []
+ for l in self.labels:
+ if l not in self.labelCodes:
+ self.labelCodes.append(l)
+ self.labelCodesCount.append(0)
+ self.labelCodesCount[self.labelCodes.index(l)] += 1
-
- def getAttributeValues(self, sampleIds, attributeId):
- vals = []
- for sid in sampleIds:
- val = self.sample[sid][attributeId]
- if val not in vals:
- vals.append(val)
- # print(vals)
- return vals
+ def getLabelCodeId(self, sampleId):
+ return self.labelCodes.index(self.labels[sampleId])
-
- def getEntropy(self, sampleIds):
- entropy = 0
- labelCount = [0] * len(self.labelCodes)
- for sid in sampleIds:
- labelCount[self.getLabelCodeId(sid)] += 1
- # print("-ge", labelCount)
- for lv in labelCount:
- # print(lv)
- if lv != 0:
- entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
- else:
- entropy += 0
- return entropy
+ def getAttributeValues(self, sampleIds, attributeId):
+ vals = []
+ for sid in sampleIds:
+ val = self.sample[sid][attributeId]
+ if val not in vals:
+ vals.append(val)
+ # print(vals)
+ return vals
-
- def getDominantLabel(self, sampleIds):
- labelCodesCount = [0] * len(self.labelCodes)
- for sid in sampleIds:
- labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
- return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
+ def getEntropy(self, sampleIds):
+ entropy = 0
+ labelCount = [0] * len(self.labelCodes)
+ for sid in sampleIds:
+ labelCount[self.getLabelCodeId(sid)] += 1
+ # print("-ge", labelCount)
+ for lv in labelCount:
+ # print(lv)
+ if lv != 0:
+ entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
+ else:
+ entropy += 0
+ return entropy
-
- def getInformationGain(self, sampleIds, attributeId):
- gain = self.getEntropy(sampleIds)
- attributeVals = []
- attributeValsCount = []
- attributeValsIds = []
- for sid in sampleIds:
- val = self.sample[sid][attributeId]
- if val not in attributeVals:
- attributeVals.append(val)
- attributeValsCount.append(0)
- attributeValsIds.append([])
- vid = attributeVals.index(val)
- attributeValsCount[vid] += 1
- attributeValsIds[vid].append(sid)
- # print("-gig", self.attributes[attributeId])
- for vc, vids in zip(attributeValsCount, attributeValsIds):
- # print("-gig", vids)
- gain -= vc/len(sampleIds) * self.getEntropy(vids)
- return gain
+ def getDominantLabel(self, sampleIds):
+ labelCodesCount = [0] * len(self.labelCodes)
+ for sid in sampleIds:
+ labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
+ return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
-
- def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
- attributesEntropy = [0] * len(attributeIds)
- for i, attId in zip(range(len(attributeIds)), attributeIds):
- attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
- maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
- return self.attributes[maxId], maxId
+ def getInformationGain(self, sampleIds, attributeId):
+ gain = self.getEntropy(sampleIds)
+ attributeVals = []
+ attributeValsCount = []
+ attributeValsIds = []
+ for sid in sampleIds:
+ val = self.sample[sid][attributeId]
+ if val not in attributeVals:
+ attributeVals.append(val)
+ attributeValsCount.append(0)
+ attributeValsIds.append([])
+ vid = attributeVals.index(val)
+ attributeValsCount[vid] += 1
+ attributeValsIds[vid].append(sid)
+ # print("-gig", self.attributes[attributeId])
+ for vc, vids in zip(attributeValsCount, attributeValsIds):
+ # print("-gig", vids)
+ gain -= vc/len(sampleIds) * self.getEntropy(vids)
+ return gain
-
- def isSingleLabeled(self, sampleIds):
- label = self.labels[sampleIds[0]]
- for sid in sampleIds:
- if self.labels[sid] != label:
- return False
- return True
+ def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
+ attributesEntropy = [0] * len(attributeIds)
+ for i, attId in zip(range(len(attributeIds)), attributeIds):
+ attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
+ maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
+ return self.attributes[maxId], maxId
-
- def getLabel(self, sampleId):
- return self.labels[sampleId]
+ def isSingleLabeled(self, sampleIds):
+ label = self.labels[sampleIds[0]]
+ for sid in sampleIds:
+ if self.labels[sid] != label:
+ return False
+ return True
-
- def id3(self):
- sampleIds = [x for x in range(len(self.sample))]
- attributeIds = [x for x in range(len(self.attributes))]
- self.root = self.id3Recv(sampleIds, attributeIds, self.root)
+ def getLabel(self, sampleId):
+ return self.labels[sampleId]
-
- def id3Recv(self, sampleIds, attributeIds, root):
- root = Node() # Initialize current root
- if self.isSingleLabeled(sampleIds):
- root.value = self.labels[sampleIds[0]]
- return root
- # print(attributeIds)
- if len(attributeIds) == 0:
- root.value = self.getDominantLabel(sampleIds)
- return root
- bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
+ def id3(self):
+ sampleIds = [x for x in range(len(self.sample))]
+ attributeIds = [x for x in range(len(self.attributes))]
+ self.root = self.id3Recv(sampleIds, attributeIds, self.root)
+
+ def id3Recv(self, sampleIds, attributeIds, root):
+ root = Node() # Initialize current root
+ if self.isSingleLabeled(sampleIds):
+ root.value = self.labels[sampleIds[0]]
+ return root
+ # print(attributeIds)
+ if len(attributeIds) == 0:
+ root.value = self.getDominantLabel(sampleIds)
+ return root
+ bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
sampleIds, attributeIds)
- # print(bestAttrName)
- root.value = bestAttrName
- root.childs = [] # Create list of children
- for value in self.getAttributeValues(sampleIds, bestAttrId):
- # print(value)
- child = Node()
- child.value = value
- root.childs.append(child) # Append new child node to current
- # root
- childSampleIds = []
- for sid in sampleIds:
- if self.sample[sid][bestAttrId] == value:
- childSampleIds.append(sid)
- if len(childSampleIds) == 0:
- child.next = self.getDominantLabel(sampleIds)
- else:
- # print(bestAttrName, bestAttrId)
- # print(attributeIds)
- if len(attributeIds) > 0 and bestAttrId in attributeIds:
- toRemove = attributeIds.index(bestAttrId)
- attributeIds.pop(toRemove)
- child.next = self.id3Recv(
- childSampleIds, attributeIds, child.next)
- return root
+ # print(bestAttrName)
+ root.value = bestAttrName
+ root.childs = [] # Create list of children
+ for value in self.getAttributeValues(sampleIds, bestAttrId):
+ # print(value)
+ child = Node()
+ child.value = value
+ root.childs.append(child) # Append new child node to current
+ # root
+ childSampleIds = []
+ for sid in sampleIds:
+ if self.sample[sid][bestAttrId] == value:
+ childSampleIds.append(sid)
+ if len(childSampleIds) == 0:
+ child.next = self.getDominantLabel(sampleIds)
+ else:
+ # print(bestAttrName, bestAttrId)
+ # print(attributeIds)
+ if len(attributeIds) > 0 and bestAttrId in attributeIds:
+ toRemove = attributeIds.index(bestAttrId)
+ attributeIds.pop(toRemove)
+ child.next = self.id3Recv(
+ childSampleIds, attributeIds, child.next)
+ return root
-
- def printTree(self):
- if self.root:
- roots = deque()
- roots.append(self.root)
- while len(roots) > 0:
- root = roots.popleft()
- print(root.value)
- if root.childs:
- for child in root.childs:
- print('({})'.format(child.value))
- roots.append(child.next)
- elif root.next:
- print(root.next)
+ def printTree(self):
+ if self.root:
+ roots = deque()
+ roots.append(self.root)
+ while len(roots) > 0:
+ root = roots.popleft()
+ print(root.value)
+ if root.childs:
+ for child in root.childs:
+ print('({})'.format(child.value))
+ roots.append(child.next)
+ elif root.next:
+ print(root.next)
-
-def test():
- f = open('DataFiles/rideclass.csv')
- attributes = f.readline().split(',')
- attributes = attributes[1:len(attributes)-1]
- print(attributes)
- sample = f.readlines()
- f.close()
- for i in range(len(sample)):
- sample[i] = re.sub('\d+,', '', sample[i])
- sample[i] = sample[i].strip().split(',')
- labels = []
- for s in sample:
- labels.append(s.pop())
- # print(sample)
- # print(labels)
- decisionTree = DecisionTree(sample, attributes, labels)
- print("System entropy {}".format(decisionTree.entropy))
- decisionTree.id3()
- decisionTree.printTree()
-
-if __name__ == '__main__':
+def test():
+ f = open('DataFiles/rideclass.csv')
+ attributes = f.readline().split(',')
+ attributes = attributes[1:len(attributes)-1]
+ print(attributes)
+ sample = f.readlines()
+ f.close()
+ for i in range(len(sample)):
+ sample[i] = re.sub('\d+,', '', sample[i])
+ sample[i] = sample[i].strip().split(',')
+ labels = []
+ for s in sample:
+ labels.append(s.pop())
+ # print(sample)
+ # print(labels)
+ decisionTree = DecisionTree(sample, attributes, labels)
+ print("System entropy {}".format(decisionTree.entropy))
+ decisionTree.id3()
+ decisionTree.printTree()
+
+
+if __name__ == '__main__':
test()
-
+
-
Cancer Data again now with Decision Trees and other Methods
+Cancer Data again now with Decision Trees and other Methods
@@ -1160,7 +1131,7 @@ deep_tree_clf.fit(X_train_scaled, y_train)
-
Another example, the moons again
+Another example, the moons again
@@ -1232,7 +1203,7 @@ plt.show()
-
Playing around with regions
+Playing around with regions
@@ -1260,7 +1231,7 @@ plt.show()
-
Regression trees
+Regression trees
@@ -1282,7 +1253,7 @@ tree_reg.fit(X, y)
-
Final regressor code
+Final regressor code
@@ -1360,7 +1331,7 @@ plt.show()
-
Pros and cons of trees, pros
+Pros and cons of trees, pros
- White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines)
@@ -1374,7 +1345,7 @@ plt.show()
-Disadvantages
+Disadvantages
- Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches
@@ -1391,7 +1362,7 @@ However, by aggregating many decision trees, using methods like bagging, random
-
Bagging
+Bagging
The plain decision trees suffer from high
@@ -1410,7 +1381,7 @@ learning method.
-
More bagging
+More bagging
Bagging typically results in improved accuracy
@@ -1439,7 +1410,7 @@ predictor, averaged over all \( B \) trees.
-
Simple example, head or tail
+Simple example, head or tail
@@ -1459,7 +1430,7 @@ plt.show()
-
Bagging Example
+Bagging Example
@@ -1510,7 +1481,7 @@ voting_clf.fit(X_train, y_train)
-
Random forests
+Random forests
Random forests provide an improvement over bagged trees by way of a
@@ -1554,7 +1525,7 @@ this setting.
-
A simple scikit-learn example
+A simple scikit-learn example
@@ -1572,7 +1543,7 @@ accuracy = cross_validate(Random_Forest_mode
-
Please, not the moons again!
+Please, not the moons again!
@@ -1630,7 +1601,7 @@ voting_clf.fit(X_train, y_train)
-
Bagging examples
+Bagging examples
@@ -1691,7 +1662,7 @@ plt.show()
-
Then random forests
+Then random forests
diff --git a/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb b/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
index e06ed31ed..41458d8a9 100644
--- a/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
+++ b/doc/pub/DecisionTrees/ipynb/DecisionTrees.ipynb
@@ -47,12 +47,8 @@
"\n",
"## A typical Decision Tree with its pertinent Jargon, Classification Problem\n",
"\n",
- "In the figure here we present a decision tree obtained from a classification problem\n",
"\n",
"\n",
- "## A typical Decision Tree with its pertinent Jargon, Regeression Problem\n",
- "\n",
- "In the figure we present a decision tree obtained from a simple regression problem\n",
"\n",
"\n",
"## General Features\n",
@@ -818,17 +814,26 @@
"The ID3 algorithm uses this information gain measure to select among the candidate\n",
"attributes at each step while growing the tree.\n",
"\n",
- "## Implementing the ID3 Algorithm\n",
- "\n",
+ "## Implementing the ID3 Algorithm"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
"import re\n",
"import math\n",
"from collections import deque\n",
"\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
+ "# x is examples in training set\n",
+ "# y is set of targets\n",
+ "# label is target attributes\n",
+ "# Node is a class which has properties values, childs, and next\n",
+ "# root is top node in the decision tree\n",
"\n",
"class Node(object):\n",
"\tdef __init__(self):\n",
@@ -836,8 +841,8 @@
"\t\tself.next = None\n",
"\t\tself.childs = None\n",
"\n",
- "\n",
- "\n",
+ "# Simple class of Decision Tree\n",
+ "# Aimed for who want to learn Decision Tree, so it is not optimized\n",
"class DecisionTree(object):\n",
"\tdef __init__(self, sample, attributes, labels):\n",
"\t\tself.sample = sample\n",
@@ -1006,15 +1011,19 @@
"\n",
"\n",
"if __name__ == '__main__':\n",
- "\ttest()\n",
- "\n",
- "\n",
+ "\ttest()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
"## Cancer Data again now with Decision Trees and other Methods"
]
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 7,
"metadata": {
"collapsed": false
},
@@ -1072,7 +1081,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 8,
"metadata": {
"collapsed": false
},
@@ -1153,7 +1162,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 9,
"metadata": {
"collapsed": false
},
@@ -1190,7 +1199,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 10,
"metadata": {
"collapsed": false
},
@@ -1206,7 +1215,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 11,
"metadata": {
"collapsed": false
},
@@ -1227,7 +1236,7 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 12,
"metadata": {
"collapsed": false
},
@@ -1275,7 +1284,7 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 13,
"metadata": {
"collapsed": false
},
@@ -1393,7 +1402,7 @@
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": 14,
"metadata": {
"collapsed": false
},
@@ -1422,7 +1431,7 @@
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": 15,
"metadata": {
"collapsed": false
},
@@ -1528,7 +1537,7 @@
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": 16,
"metadata": {
"collapsed": false
},
@@ -1555,7 +1564,7 @@
},
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": 17,
"metadata": {
"collapsed": false
},
@@ -1583,7 +1592,7 @@
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": 18,
"metadata": {
"collapsed": false
},
@@ -1599,7 +1608,7 @@
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": 19,
"metadata": {
"collapsed": false
},
@@ -1617,7 +1626,7 @@
},
{
"cell_type": "code",
- "execution_count": 19,
+ "execution_count": 20,
"metadata": {
"collapsed": false
},
@@ -1640,7 +1649,7 @@
},
{
"cell_type": "code",
- "execution_count": 20,
+ "execution_count": 21,
"metadata": {
"collapsed": false
},
@@ -1658,7 +1667,7 @@
},
{
"cell_type": "code",
- "execution_count": 21,
+ "execution_count": 22,
"metadata": {
"collapsed": false
},
@@ -1670,7 +1679,7 @@
},
{
"cell_type": "code",
- "execution_count": 22,
+ "execution_count": 23,
"metadata": {
"collapsed": false
},
@@ -1684,7 +1693,7 @@
},
{
"cell_type": "code",
- "execution_count": 23,
+ "execution_count": 24,
"metadata": {
"collapsed": false
},
@@ -1727,7 +1736,7 @@
},
{
"cell_type": "code",
- "execution_count": 24,
+ "execution_count": 25,
"metadata": {
"collapsed": false
},
@@ -1740,7 +1749,7 @@
},
{
"cell_type": "code",
- "execution_count": 25,
+ "execution_count": 26,
"metadata": {
"collapsed": false
},
diff --git a/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz b/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz
index 9ecef5eea..0cb193e00 100644
Binary files a/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz and b/doc/pub/DecisionTrees/ipynb/ipynb-DecisionTrees-src.tar.gz differ
diff --git a/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf b/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf
index 445baf294..8adec554f 100644
Binary files a/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf and b/doc/pub/DecisionTrees/pdf/DecisionTrees-minted.pdf differ
diff --git a/doc/src/DecisionTrees/DecisionTrees.do.txt b/doc/src/DecisionTrees/DecisionTrees.do.txt
index abcc176f6..b1286f7b2 100644
--- a/doc/src/DecisionTrees/DecisionTrees.do.txt
+++ b/doc/src/DecisionTrees/DecisionTrees.do.txt
@@ -35,13 +35,8 @@ given some assumptions, make predictions about the target feature value
!split
===== A typical Decision Tree with its pertinent Jargon, Classification Problem =====
-In the figure here we present a decision tree obtained from a classification problem
-!split
-===== A typical Decision Tree with its pertinent Jargon, Regeression Problem =====
-
-In the figure we present a decision tree obtained from a simple regression problem
!split
@@ -665,12 +660,13 @@ attributes at each step while growing the tree.
!split
===== Implementing the ID3 Algorithm =====
+!bc pycod
import re
import math
from collections import deque
# x is examples in training set
-# y is set of attributes
+# y is set of targets
# label is target attributes
# Node is a class which has properties values, childs, and next
# root is top node in the decision tree
@@ -852,7 +848,7 @@ def test():
if __name__ == '__main__':
test()
-
+!ec
!split
===== Cancer Data again now with Decision Trees and other Methods =====