update on decision trees
This commit is contained in:
mhjensen
@@ -0,0 +1,57 @@
|
||||
digraph Tree {
|
||||
node [shape=box, style="filled, rounded", color="black", fontname=helvetica] ;
|
||||
edge [fontname=helvetica] ;
|
||||
0 [label="worst perimeter <= 106.05\ngini = 0.465\nsamples = 426\nvalue = [[269, 157]\n[157, 269]]", fillcolor="#e5813908"] ;
|
||||
1 [label="worst concave points <= 0.159\ngini = 0.067\nsamples = 259\nvalue = [[250, 9]\n[9, 250]]", fillcolor="#e58139db"] ;
|
||||
0 -> 1 [labeldistance=2.5, labelangle=45, headlabel="True"] ;
|
||||
2 [label="worst concave points <= 0.135\ngini = 0.031\nsamples = 253\nvalue = [[249, 4]\n[4, 249]]", fillcolor="#e58139ee"] ;
|
||||
1 -> 2 ;
|
||||
3 [label="radius error <= 0.643\ngini = 0.008\nsamples = 242\nvalue = [[241, 1]\n[1, 241]]", fillcolor="#e58139fb"] ;
|
||||
2 -> 3 ;
|
||||
4 [label="gini = 0.0\nsamples = 239\nvalue = [[239, 0]\n[0, 239]]", fillcolor="#e58139ff"] ;
|
||||
3 -> 4 ;
|
||||
5 [label="worst symmetry <= 0.208\ngini = 0.444\nsamples = 3\nvalue = [[2, 1]\n[1, 2]]", fillcolor="#e5813913"] ;
|
||||
3 -> 5 ;
|
||||
6 [label="gini = 0.0\nsamples = 1\nvalue = [[0, 1]\n[1, 0]]", fillcolor="#e58139ff"] ;
|
||||
5 -> 6 ;
|
||||
7 [label="gini = 0.0\nsamples = 2\nvalue = [[2, 0]\n[0, 2]]", fillcolor="#e58139ff"] ;
|
||||
5 -> 7 ;
|
||||
8 [label="worst texture <= 29.455\ngini = 0.397\nsamples = 11\nvalue = [[8, 3]\n[3, 8]]", fillcolor="#e581392c"] ;
|
||||
2 -> 8 ;
|
||||
9 [label="gini = 0.0\nsamples = 8\nvalue = [[8, 0]\n[0, 8]]", fillcolor="#e58139ff"] ;
|
||||
8 -> 9 ;
|
||||
10 [label="gini = 0.0\nsamples = 3\nvalue = [[0, 3]\n[3, 0]]", fillcolor="#e58139ff"] ;
|
||||
8 -> 10 ;
|
||||
11 [label="mean texture <= 16.22\ngini = 0.278\nsamples = 6\nvalue = [[1, 5]\n[5, 1]]", fillcolor="#e581396b"] ;
|
||||
1 -> 11 ;
|
||||
12 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
11 -> 12 ;
|
||||
13 [label="gini = 0.0\nsamples = 5\nvalue = [[0, 5]\n[5, 0]]", fillcolor="#e58139ff"] ;
|
||||
11 -> 13 ;
|
||||
14 [label="worst texture <= 20.645\ngini = 0.202\nsamples = 167\nvalue = [[19, 148]\n[148, 19]]", fillcolor="#e5813994"] ;
|
||||
0 -> 14 [labeldistance=2.5, labelangle=-45, headlabel="False"] ;
|
||||
15 [label="worst radius <= 17.74\ngini = 0.375\nsamples = 16\nvalue = [[12, 4]\n[4, 12]]", fillcolor="#e5813938"] ;
|
||||
14 -> 15 ;
|
||||
16 [label="gini = 0.0\nsamples = 11\nvalue = [[11, 0]\n[0, 11]]", fillcolor="#e58139ff"] ;
|
||||
15 -> 16 ;
|
||||
17 [label="mean texture <= 13.745\ngini = 0.32\nsamples = 5\nvalue = [[1, 4]\n[4, 1]]", fillcolor="#e5813955"] ;
|
||||
15 -> 17 ;
|
||||
18 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
17 -> 18 ;
|
||||
19 [label="gini = 0.0\nsamples = 4\nvalue = [[0, 4]\n[4, 0]]", fillcolor="#e58139ff"] ;
|
||||
17 -> 19 ;
|
||||
20 [label="mean concave points <= 0.049\ngini = 0.088\nsamples = 151\nvalue = [[7, 144]\n[144, 7]]", fillcolor="#e58139d0"] ;
|
||||
14 -> 20 ;
|
||||
21 [label="concave points error <= 0.01\ngini = 0.48\nsamples = 15\nvalue = [[6, 9]\n[9, 6]]", fillcolor="#e5813900"] ;
|
||||
20 -> 21 ;
|
||||
22 [label="gini = 0.0\nsamples = 9\nvalue = [[0, 9]\n[9, 0]]", fillcolor="#e58139ff"] ;
|
||||
21 -> 22 ;
|
||||
23 [label="gini = 0.0\nsamples = 6\nvalue = [[6, 0]\n[0, 6]]", fillcolor="#e58139ff"] ;
|
||||
21 -> 23 ;
|
||||
24 [label="worst smoothness <= 0.096\ngini = 0.015\nsamples = 136\nvalue = [[1, 135]\n[135, 1]]", fillcolor="#e58139f7"] ;
|
||||
20 -> 24 ;
|
||||
25 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
24 -> 25 ;
|
||||
26 [label="gini = 0.0\nsamples = 135\nvalue = [[0, 135]\n[135, 0]]", fillcolor="#e58139ff"] ;
|
||||
24 -> 26 ;
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 216 KiB |
@@ -400,67 +400,179 @@ s = -\sum_{k=1}^K p_{mk}\log{p_{mk}}.
|
||||
|
||||
|
||||
!split
|
||||
===== The CART (Classification and Regression Tree) algorithm =====
|
||||
===== Visualizing the Tree, Classification =====
|
||||
!bc pycod
|
||||
import os
|
||||
from sklearn.datasets import load_breast_cancer
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.metrics import confusion_matrix
|
||||
from sklearn.tree import export_graphviz
|
||||
|
||||
The above functions (gini, entropy and misclassification error) are important components of the so-called CART algorithm. We will discuss this algorithm first before we move on to the information gain algorithm ID3.
|
||||
from IPython.display import Image
|
||||
from pydot import graph_from_dot_data
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
|
||||
cancer = load_breast_cancer()
|
||||
X = pd.DataFrame(cancer.data, columns=cancer.feature_names)
|
||||
print(X)
|
||||
y = pd.Categorical.from_codes(cancer.target, cancer.target_names)
|
||||
y = pd.get_dummies(y)
|
||||
print(y)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)
|
||||
tree_clf = DecisionTreeClassifier(max_depth=5)
|
||||
tree_clf.fit(X_train, y_train)
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file="DataFiles/cancer.dot",
|
||||
feature_names=cancer.feature_names,
|
||||
class_names=cancer.target_names,
|
||||
rounded=True,
|
||||
filled=True
|
||||
)
|
||||
cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
|
||||
os.system(cmd)
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Visualizing the Tree, The Moons =====
|
||||
!bc pycod
|
||||
# Common imports
|
||||
import numpy as np
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.datasets import make_moons
|
||||
from sklearn.tree import export_graphviz
|
||||
from pydot import graph_from_dot_data
|
||||
import pandas as pd
|
||||
import os
|
||||
|
||||
np.random.seed(42)
|
||||
X, y = make_moons(n_samples=100, noise=0.25, random_state=53)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)
|
||||
tree_clf = DecisionTreeClassifier(max_depth=5)
|
||||
tree_clf.fit(X_train, y_train)
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file="DataFiles/moons.dot",
|
||||
rounded=True,
|
||||
filled=True
|
||||
)
|
||||
cmd = 'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png'
|
||||
os.system(cmd)
|
||||
!ec
|
||||
|
||||
|
||||
!split
|
||||
===== Computing the Gini index =====
|
||||
|
||||
The example we will look at is a classical one in many Machine
|
||||
Learning applications. Based on various meteorological features, we
|
||||
have several so-called attributes which decide whether we at the end
|
||||
will do some outdoor activity like skiing, going for a bike ride etc
|
||||
etc. The table here contains the feautures _outlook_, _temperature_,
|
||||
_humidity_ and _wind_. The target or output is whether we ride
|
||||
(True=1) or whether we do something else that day (False=0). The
|
||||
attributes for each feature are then sunny, overcast and rain for the
|
||||
outlook, hot, cold and mild for temperature, high and normal for
|
||||
humidity and weak and strong for wind.
|
||||
|
||||
The table here summarizes the various attributes and
|
||||
|-------------------------------------------|
|
||||
|Day| Outlook |Temperature | Humidity | Wind | Ride|
|
||||
|-------------------------------------------|
|
||||
|1 | Sunny | Hot | High | Weak | 0 |
|
||||
|2 | Sunny | Hot | High | Strong | 1 |
|
||||
|3 | Overcast | Hot | High | Weak | 1 |
|
||||
|4 | Rain | Mild | High | Weak | 1 |
|
||||
|5 | Rain | Cool | Normal | Weak | 1 |
|
||||
|6 | Rain | Cool | Normal | Strong | 0 |
|
||||
|7 | Overcast | Cool | Normal | Strong | 1 |
|
||||
|8 | Sunny | Mild | High | Weak | 0 |
|
||||
|9 | Sunny | Cool | Normal | Weak | 1 |
|
||||
|10 | Rain | Mild | Normal | Weak | 1 |
|
||||
|11 | Sunny | Mild | Normal | Strong | 1 |
|
||||
|12 | Overcast | Mild | High | Strong | 1 |
|
||||
|13 | Overcast | Hot | Normal | Weak | 1 |
|
||||
|14 | Rain | Mild | High | Strong | 0 |
|
||||
|-------------------------------------------|
|
||||
|
||||
|
||||
!split
|
||||
===== Simple Python Code to read in Data =====
|
||||
|
||||
!bc pycod
|
||||
# Common imports
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.preprocessing import StandardScaler, OneHotEncoder
|
||||
from sklearn.compose import ColumnTransformer
|
||||
from IPython.display import Image
|
||||
from pydot import graph_from_dot_data
|
||||
import os
|
||||
|
||||
# Where to save the figures and data files
|
||||
PROJECT_ROOT_DIR = "Results"
|
||||
FIGURE_ID = "Results/FigureFiles"
|
||||
DATA_ID = "DataFiles/"
|
||||
|
||||
if not os.path.exists(PROJECT_ROOT_DIR):
|
||||
os.mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
if not os.path.exists(FIGURE_ID):
|
||||
os.makedirs(FIGURE_ID)
|
||||
|
||||
if not os.path.exists(DATA_ID):
|
||||
os.makedirs(DATA_ID)
|
||||
|
||||
def image_path(fig_id):
|
||||
return os.path.join(FIGURE_ID, fig_id)
|
||||
|
||||
def data_path(dat_id):
|
||||
return os.path.join(DATA_ID, dat_id)
|
||||
|
||||
def save_fig(fig_id):
|
||||
plt.savefig(image_path(fig_id) + ".png", format='png')
|
||||
|
||||
infile = open(data_path("ride.csv"),'r')
|
||||
|
||||
# Read the experimental data with Pandas
|
||||
from IPython.display import display
|
||||
ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
|
||||
ridedata = pd.DataFrame(ridedata)
|
||||
display(ridedata)
|
||||
# Features and targets
|
||||
X = ridedata.loc[:, ridedata.columns != 'Ride'].values
|
||||
display(X)
|
||||
y = ridedata.loc[:, ridedata.columns == 'Ride'].values
|
||||
display(y)
|
||||
# Categorical variables to one-hot's
|
||||
onehotencoder = OneHotEncoder(categories="auto")
|
||||
|
||||
X = ColumnTransformer([("", onehotencoder)]).fit_transform(X)
|
||||
y.shape
|
||||
|
||||
display(X)
|
||||
display(y)
|
||||
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Computing the Gini Factor =====
|
||||
|
||||
The above functions (gini, entropy and misclassification error) are
|
||||
important components of the so-called CART algorithm. We will discuss
|
||||
this algorithm below after we have discussed the information gain
|
||||
algorithm ID3.
|
||||
|
||||
In the example here we have converted all our attributes into numerical values $0,1,2$ etc.
|
||||
|
||||
!bc pycod
|
||||
from random import seed
|
||||
from random import randrange
|
||||
from csv import reader
|
||||
|
||||
# Load a CSV file
|
||||
def load_csv(filename):
|
||||
file = open(filename, "rb")
|
||||
lines = reader(file)
|
||||
dataset = list(lines)
|
||||
return dataset
|
||||
|
||||
# Convert string column to float
|
||||
def str_column_to_float(dataset, column):
|
||||
for row in dataset:
|
||||
row[column] = float(row[column].strip())
|
||||
|
||||
# Split a dataset into k folds
|
||||
def cross_validation_split(dataset, n_folds):
|
||||
dataset_split = list()
|
||||
dataset_copy = list(dataset)
|
||||
fold_size = int(len(dataset) / n_folds)
|
||||
for i in range(n_folds):
|
||||
fold = list()
|
||||
while len(fold) < fold_size:
|
||||
index = randrange(len(dataset_copy))
|
||||
fold.append(dataset_copy.pop(index))
|
||||
dataset_split.append(fold)
|
||||
return dataset_split
|
||||
|
||||
# Calculate accuracy percentage
|
||||
def accuracy_metric(actual, predicted):
|
||||
correct = 0
|
||||
for i in range(len(actual)):
|
||||
if actual[i] == predicted[i]:
|
||||
correct += 1
|
||||
return correct / float(len(actual)) * 100.0
|
||||
|
||||
# Evaluate an algorithm using a cross validation split
|
||||
def evaluate_algorithm(dataset, algorithm, n_folds, *args):
|
||||
folds = cross_validation_split(dataset, n_folds)
|
||||
scores = list()
|
||||
for fold in folds:
|
||||
train_set = list(folds)
|
||||
train_set.remove(fold)
|
||||
train_set = sum(train_set, [])
|
||||
test_set = list()
|
||||
for row in fold:
|
||||
row_copy = list(row)
|
||||
test_set.append(row_copy)
|
||||
row_copy[-1] = None
|
||||
predicted = algorithm(train_set, test_set, *args)
|
||||
actual = [row[-1] for row in fold]
|
||||
accuracy = accuracy_metric(actual, predicted)
|
||||
scores.append(accuracy)
|
||||
return scores
|
||||
|
||||
# Split a dataset based on an attribute and an attribute value
|
||||
def test_split(index, value, dataset):
|
||||
left, right = list(), list()
|
||||
@@ -490,7 +602,7 @@ def gini_index(groups, classes):
|
||||
# weight the group score by its relative size
|
||||
gini += (1.0 - score) * (size / n_instances)
|
||||
return gini
|
||||
|
||||
|
||||
# Select the best split point for a dataset
|
||||
def get_split(dataset):
|
||||
class_values = list(set(row[-1] for row in dataset))
|
||||
@@ -499,84 +611,28 @@ def get_split(dataset):
|
||||
for row in dataset:
|
||||
groups = test_split(index, row[index], dataset)
|
||||
gini = gini_index(groups, class_values)
|
||||
print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini))
|
||||
if gini < b_score:
|
||||
b_index, b_value, b_score, b_groups = index, row[index], gini, groups
|
||||
return {'index':b_index, 'value':b_value, 'groups':b_groups}
|
||||
|
||||
# Create a terminal node value
|
||||
def to_terminal(group):
|
||||
outcomes = [row[-1] for row in group]
|
||||
return max(set(outcomes), key=outcomes.count)
|
||||
|
||||
# Create child splits for a node or make terminal
|
||||
def split(node, max_depth, min_size, depth):
|
||||
left, right = node['groups']
|
||||
del(node['groups'])
|
||||
# check for a no split
|
||||
if not left or not right:
|
||||
node['left'] = node['right'] = to_terminal(left + right)
|
||||
return
|
||||
# check for max depth
|
||||
if depth >= max_depth:
|
||||
node['left'], node['right'] = to_terminal(left), to_terminal(right)
|
||||
return
|
||||
# process left child
|
||||
if len(left) <= min_size:
|
||||
node['left'] = to_terminal(left)
|
||||
else:
|
||||
node['left'] = get_split(left)
|
||||
split(node['left'], max_depth, min_size, depth+1)
|
||||
# process right child
|
||||
if len(right) <= min_size:
|
||||
node['right'] = to_terminal(right)
|
||||
else:
|
||||
node['right'] = get_split(right)
|
||||
split(node['right'], max_depth, min_size, depth+1)
|
||||
|
||||
# Build a decision tree
|
||||
def build_tree(train, max_depth, min_size):
|
||||
root = get_split(train)
|
||||
split(root, max_depth, min_size, 1)
|
||||
return root
|
||||
|
||||
# Make a prediction with a decision tree
|
||||
def predict(node, row):
|
||||
if row[node['index']] < node['value']:
|
||||
if isinstance(node['left'], dict):
|
||||
return predict(node['left'], row)
|
||||
else:
|
||||
return node['left']
|
||||
else:
|
||||
if isinstance(node['right'], dict):
|
||||
return predict(node['right'], row)
|
||||
else:
|
||||
return node['right']
|
||||
|
||||
# Classification and Regression Tree Algorithm
|
||||
def decision_tree(train, test, max_depth, min_size):
|
||||
tree = build_tree(train, max_depth, min_size)
|
||||
predictions = list()
|
||||
for row in test:
|
||||
prediction = predict(tree, row)
|
||||
predictions.append(prediction)
|
||||
return(predictions)
|
||||
|
||||
# Test CART
|
||||
seed(1)
|
||||
# load and prepare data
|
||||
filename = 'DataFiles/rideclass.csv'
|
||||
dataset = load_csv(filename)
|
||||
# convert string attributes to integers
|
||||
for i in range(len(dataset[0])):
|
||||
str_column_to_float(dataset, i)
|
||||
# evaluate algorithm
|
||||
n_folds = 5
|
||||
max_depth = 5
|
||||
min_size = 10
|
||||
scores = evaluate_algorithm(dataset, decision_tree, n_folds, max_depth, min_size)
|
||||
print('Scores: %s' % scores)
|
||||
print('Mean Accuracy: %.3f%%' % (sum(scores)/float(len(scores))))
|
||||
dataset = [[0,0,0,0,0],
|
||||
[0,0,0,1,1],
|
||||
[1,0,0,0,1],
|
||||
[2,1,0,0,1],
|
||||
[2,2,1,0,1],
|
||||
[2,2,1,1,0],
|
||||
[1,2,1,1,1],
|
||||
[0,1,0,0,0],
|
||||
[0,2,1,0,1],
|
||||
[2,1,1,0,1],
|
||||
[0,1,1,1,1],
|
||||
[1,1,0,1,1],
|
||||
[1,0,1,0,1],
|
||||
[2,1,0,1,0]]
|
||||
|
||||
split = get_split(dataset)
|
||||
print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
|
||||
!ec
|
||||
|
||||
!split
|
||||
@@ -609,10 +665,197 @@ attributes at each step while growing the tree.
|
||||
!split
|
||||
===== Implementing the ID3 Algorithm =====
|
||||
|
||||
_more text to come here_, material presented during lecture Friday Oct 25.
|
||||
import re
|
||||
import math
|
||||
from collections import deque
|
||||
|
||||
# x is examples in training set
|
||||
# y is set of attributes
|
||||
# label is target attributes
|
||||
# Node is a class which has properties values, childs, and next
|
||||
# root is top node in the decision tree
|
||||
|
||||
class Node(object):
|
||||
def __init__(self):
|
||||
self.value = None
|
||||
self.next = None
|
||||
self.childs = None
|
||||
|
||||
# Simple class of Decision Tree
|
||||
# Aimed for who want to learn Decision Tree, so it is not optimized
|
||||
class DecisionTree(object):
|
||||
def __init__(self, sample, attributes, labels):
|
||||
self.sample = sample
|
||||
self.attributes = attributes
|
||||
self.labels = labels
|
||||
self.labelCodes = None
|
||||
self.labelCodesCount = None
|
||||
self.initLabelCodes()
|
||||
# print(self.labelCodes)
|
||||
self.root = None
|
||||
self.entropy = self.getEntropy([x for x in range(len(self.labels))])
|
||||
|
||||
def initLabelCodes(self):
|
||||
self.labelCodes = []
|
||||
self.labelCodesCount = []
|
||||
for l in self.labels:
|
||||
if l not in self.labelCodes:
|
||||
self.labelCodes.append(l)
|
||||
self.labelCodesCount.append(0)
|
||||
self.labelCodesCount[self.labelCodes.index(l)] += 1
|
||||
|
||||
def getLabelCodeId(self, sampleId):
|
||||
return self.labelCodes.index(self.labels[sampleId])
|
||||
|
||||
def getAttributeValues(self, sampleIds, attributeId):
|
||||
vals = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in vals:
|
||||
vals.append(val)
|
||||
# print(vals)
|
||||
return vals
|
||||
|
||||
def getEntropy(self, sampleIds):
|
||||
entropy = 0
|
||||
labelCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCount[self.getLabelCodeId(sid)] += 1
|
||||
# print("-ge", labelCount)
|
||||
for lv in labelCount:
|
||||
# print(lv)
|
||||
if lv != 0:
|
||||
entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
|
||||
else:
|
||||
entropy += 0
|
||||
return entropy
|
||||
|
||||
def getDominantLabel(self, sampleIds):
|
||||
labelCodesCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
|
||||
return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
|
||||
|
||||
def getInformationGain(self, sampleIds, attributeId):
|
||||
gain = self.getEntropy(sampleIds)
|
||||
attributeVals = []
|
||||
attributeValsCount = []
|
||||
attributeValsIds = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in attributeVals:
|
||||
attributeVals.append(val)
|
||||
attributeValsCount.append(0)
|
||||
attributeValsIds.append([])
|
||||
vid = attributeVals.index(val)
|
||||
attributeValsCount[vid] += 1
|
||||
attributeValsIds[vid].append(sid)
|
||||
# print("-gig", self.attributes[attributeId])
|
||||
for vc, vids in zip(attributeValsCount, attributeValsIds):
|
||||
# print("-gig", vids)
|
||||
gain -= vc/len(sampleIds) * self.getEntropy(vids)
|
||||
return gain
|
||||
|
||||
def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
|
||||
attributesEntropy = [0] * len(attributeIds)
|
||||
for i, attId in zip(range(len(attributeIds)), attributeIds):
|
||||
attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
|
||||
maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
|
||||
return self.attributes[maxId], maxId
|
||||
|
||||
def isSingleLabeled(self, sampleIds):
|
||||
label = self.labels[sampleIds[0]]
|
||||
for sid in sampleIds:
|
||||
if self.labels[sid] != label:
|
||||
return False
|
||||
return True
|
||||
|
||||
def getLabel(self, sampleId):
|
||||
return self.labels[sampleId]
|
||||
|
||||
def id3(self):
|
||||
sampleIds = [x for x in range(len(self.sample))]
|
||||
attributeIds = [x for x in range(len(self.attributes))]
|
||||
self.root = self.id3Recv(sampleIds, attributeIds, self.root)
|
||||
|
||||
def id3Recv(self, sampleIds, attributeIds, root):
|
||||
root = Node() # Initialize current root
|
||||
if self.isSingleLabeled(sampleIds):
|
||||
root.value = self.labels[sampleIds[0]]
|
||||
return root
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) == 0:
|
||||
root.value = self.getDominantLabel(sampleIds)
|
||||
return root
|
||||
bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
|
||||
sampleIds, attributeIds)
|
||||
# print(bestAttrName)
|
||||
root.value = bestAttrName
|
||||
root.childs = [] # Create list of children
|
||||
for value in self.getAttributeValues(sampleIds, bestAttrId):
|
||||
# print(value)
|
||||
child = Node()
|
||||
child.value = value
|
||||
root.childs.append(child) # Append new child node to current
|
||||
# root
|
||||
childSampleIds = []
|
||||
for sid in sampleIds:
|
||||
if self.sample[sid][bestAttrId] == value:
|
||||
childSampleIds.append(sid)
|
||||
if len(childSampleIds) == 0:
|
||||
child.next = self.getDominantLabel(sampleIds)
|
||||
else:
|
||||
# print(bestAttrName, bestAttrId)
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) > 0 and bestAttrId in attributeIds:
|
||||
toRemove = attributeIds.index(bestAttrId)
|
||||
attributeIds.pop(toRemove)
|
||||
child.next = self.id3Recv(
|
||||
childSampleIds, attributeIds, child.next)
|
||||
return root
|
||||
|
||||
def printTree(self):
|
||||
if self.root:
|
||||
roots = deque()
|
||||
roots.append(self.root)
|
||||
while len(roots) > 0:
|
||||
root = roots.popleft()
|
||||
print(root.value)
|
||||
if root.childs:
|
||||
for child in root.childs:
|
||||
print('({})'.format(child.value))
|
||||
roots.append(child.next)
|
||||
elif root.next:
|
||||
print(root.next)
|
||||
|
||||
|
||||
def test():
|
||||
f = open('DataFiles/rideclass.csv')
|
||||
attributes = f.readline().split(',')
|
||||
attributes = attributes[1:len(attributes)-1]
|
||||
print(attributes)
|
||||
sample = f.readlines()
|
||||
f.close()
|
||||
for i in range(len(sample)):
|
||||
sample[i] = re.sub('\d+,', '', sample[i])
|
||||
sample[i] = sample[i].strip().split(',')
|
||||
labels = []
|
||||
for s in sample:
|
||||
labels.append(s.pop())
|
||||
# print(sample)
|
||||
# print(labels)
|
||||
decisionTree = DecisionTree(sample, attributes, labels)
|
||||
print("System entropy {}".format(decisionTree.entropy))
|
||||
decisionTree.id3()
|
||||
decisionTree.printTree()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
|
||||
|
||||
!split
|
||||
===== Cancer Data again now with Decision Trees =====
|
||||
===== Cancer Data again now with Decision Trees and other Methods =====
|
||||
!bc pycod
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
@@ -736,7 +979,7 @@ np.random.seed(6)
|
||||
Xs = np.random.rand(100, 2) - 0.5
|
||||
ys = (Xs[:, 0] > 0).astype(np.float32) * 2
|
||||
|
||||
angle = np.pi / 4
|
||||
angle = np.pi/4
|
||||
rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
|
||||
Xsr = Xs.dot(rotation_matrix)
|
||||
|
||||
@@ -889,6 +1132,9 @@ general-purpose procedure for reducing the variance of a statistical
|
||||
learning method.
|
||||
|
||||
|
||||
!split
|
||||
===== More bagging =====
|
||||
|
||||
Bagging typically results in improved accuracy
|
||||
over prediction using a single tree. Unfortunately, however, it can be
|
||||
difficult to interpret the resulting model. Recall that one of the
|
||||
@@ -929,6 +1175,56 @@ plt.show()
|
||||
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Bagging Example =====
|
||||
!bc pycod
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.datasets import make_moons
|
||||
|
||||
X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
|
||||
|
||||
from sklearn.ensemble import RandomForestClassifier
|
||||
from sklearn.ensemble import VotingClassifier
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.svm import SVC
|
||||
|
||||
log_clf = LogisticRegression(solver="liblinear", random_state=42)
|
||||
rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
|
||||
svm_clf = SVC(gamma="auto", random_state=42)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
|
||||
voting='hard')
|
||||
|
||||
voting_clf.fit(X_train, y_train)
|
||||
|
||||
from sklearn.metrics import accuracy_score
|
||||
|
||||
for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
|
||||
|
||||
log_clf = LogisticRegression(solver="liblinear", random_state=42)
|
||||
rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
|
||||
svm_clf = SVC(gamma="auto", probability=True, random_state=42)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
|
||||
voting='soft')
|
||||
voting_clf.fit(X_train, y_train)
|
||||
|
||||
from sklearn.metrics import accuracy_score
|
||||
|
||||
for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
|
||||
|
||||
!ec
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== Random forests =====
|
||||
@@ -1113,8 +1409,4 @@ np.sum(y_pred == y_pred_rf) / len(y_pred)
|
||||
!ec
|
||||
|
||||
|
||||
!split
|
||||
===== Boosting and more =====
|
||||
More material to come here.
|
||||
|
||||
|
||||
|
||||
+4
-4
@@ -10,7 +10,7 @@ edge [fontname=helvetica] ;
|
||||
2 -> 3 ;
|
||||
4 [label="gini = 0.0\nsamples = 239\nvalue = [[239, 0]\n[0, 239]]", fillcolor="#e58139ff"] ;
|
||||
3 -> 4 ;
|
||||
5 [label="symmetry error <= 0.025\ngini = 0.444\nsamples = 3\nvalue = [[2, 1]\n[1, 2]]", fillcolor="#e5813913"] ;
|
||||
5 [label="compactness error <= 0.014\ngini = 0.444\nsamples = 3\nvalue = [[2, 1]\n[1, 2]]", fillcolor="#e5813913"] ;
|
||||
3 -> 5 ;
|
||||
6 [label="gini = 0.0\nsamples = 1\nvalue = [[0, 1]\n[1, 0]]", fillcolor="#e58139ff"] ;
|
||||
5 -> 6 ;
|
||||
@@ -22,7 +22,7 @@ edge [fontname=helvetica] ;
|
||||
8 -> 9 ;
|
||||
10 [label="gini = 0.0\nsamples = 3\nvalue = [[0, 3]\n[3, 0]]", fillcolor="#e58139ff"] ;
|
||||
8 -> 10 ;
|
||||
11 [label="worst texture <= 24.785\ngini = 0.278\nsamples = 6\nvalue = [[1, 5]\n[5, 1]]", fillcolor="#e581396b"] ;
|
||||
11 [label="mean texture <= 16.22\ngini = 0.278\nsamples = 6\nvalue = [[1, 5]\n[5, 1]]", fillcolor="#e581396b"] ;
|
||||
1 -> 11 ;
|
||||
12 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
11 -> 12 ;
|
||||
@@ -30,11 +30,11 @@ edge [fontname=helvetica] ;
|
||||
11 -> 13 ;
|
||||
14 [label="worst texture <= 20.645\ngini = 0.202\nsamples = 167\nvalue = [[19, 148]\n[148, 19]]", fillcolor="#e5813994"] ;
|
||||
0 -> 14 [labeldistance=2.5, labelangle=-45, headlabel="False"] ;
|
||||
15 [label="worst perimeter <= 116.8\ngini = 0.375\nsamples = 16\nvalue = [[12, 4]\n[4, 12]]", fillcolor="#e5813938"] ;
|
||||
15 [label="worst area <= 964.4\ngini = 0.375\nsamples = 16\nvalue = [[12, 4]\n[4, 12]]", fillcolor="#e5813938"] ;
|
||||
14 -> 15 ;
|
||||
16 [label="gini = 0.0\nsamples = 11\nvalue = [[11, 0]\n[0, 11]]", fillcolor="#e58139ff"] ;
|
||||
15 -> 16 ;
|
||||
17 [label="mean concavity <= 0.06\ngini = 0.32\nsamples = 5\nvalue = [[1, 4]\n[4, 1]]", fillcolor="#e5813955"] ;
|
||||
17 [label="worst compactness <= 0.126\ngini = 0.32\nsamples = 5\nvalue = [[1, 4]\n[4, 1]]", fillcolor="#e5813955"] ;
|
||||
15 -> 17 ;
|
||||
18 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
17 -> 18 ;
|
||||
@@ -1,3 +1,4 @@
|
||||
import os
|
||||
from sklearn.datasets import load_breast_cancer
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.model_selection import train_test_split
|
||||
@@ -9,6 +10,31 @@ from pydot import graph_from_dot_data
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
|
||||
# Where to save the figures and data files
|
||||
PROJECT_ROOT_DIR = "Results"
|
||||
FIGURE_ID = "Results/FigureFiles"
|
||||
DATA_ID = "DataFiles/"
|
||||
|
||||
if not os.path.exists(PROJECT_ROOT_DIR):
|
||||
os.mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
if not os.path.exists(FIGURE_ID):
|
||||
os.makedirs(FIGURE_ID)
|
||||
|
||||
if not os.path.exists(DATA_ID):
|
||||
os.makedirs(DATA_ID)
|
||||
|
||||
def image_path(fig_id):
|
||||
return os.path.join(FIGURE_ID, fig_id)
|
||||
|
||||
def data_path(dat_id):
|
||||
return os.path.join(DATA_ID, dat_id)
|
||||
|
||||
def save_fig(fig_id):
|
||||
plt.savefig(image_path(fig_id) + ".png", format='png')
|
||||
|
||||
|
||||
cancer = load_breast_cancer()
|
||||
X = pd.DataFrame(cancer.data, columns=cancer.feature_names)
|
||||
print(X)
|
||||
@@ -21,10 +47,11 @@ tree_clf.fit(X_train, y_train)
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file="cancer.dot",
|
||||
out_file="DataFiles/cancer.dot",
|
||||
feature_names=cancer.feature_names,
|
||||
class_names=cancer.target_names,
|
||||
rounded=True,
|
||||
filled=True
|
||||
)
|
||||
|
||||
cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
|
||||
os.system(cmd)
|
||||
|
||||
@@ -41,6 +41,7 @@ def get_split(dataset):
|
||||
b_index, b_value, b_score, b_groups = index, row[index], gini, groups
|
||||
return {'index':b_index, 'value':b_value, 'groups':b_groups}
|
||||
|
||||
"""
|
||||
dataset = [[2.771244718,1.784783929,0],
|
||||
[1.728571309,1.169761413,0],
|
||||
[3.678319846,2.81281357,0],
|
||||
@@ -51,5 +52,23 @@ dataset = [[2.771244718,1.784783929,0],
|
||||
[7.444542326,0.476683375,1],
|
||||
[10.12493903,3.234550982,1],
|
||||
[6.642287351,3.319983761,1]]
|
||||
"""
|
||||
dataset = [[0,0,0,0,0],
|
||||
[0,0,0,1,1],
|
||||
[1,0,0,0,1],
|
||||
[2,1,0,0,1],
|
||||
[2,2,1,0,1],
|
||||
[2,2,1,1,0],
|
||||
[1,2,1,1,1],
|
||||
[0,1,0,0,0],
|
||||
[0,2,1,0,1],
|
||||
[2,1,1,0,1],
|
||||
[0,1,1,1,1],
|
||||
[1,1,0,1,1],
|
||||
[1,0,1,0,1],
|
||||
[2,1,0,1,0]]
|
||||
|
||||
split = get_split(dataset)
|
||||
print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user