reshaping codes
This commit is contained in:
mhjensen
@@ -0,0 +1,57 @@
|
||||
digraph Tree {
|
||||
node [shape=box, style="filled, rounded", color="black", fontname=helvetica] ;
|
||||
edge [fontname=helvetica] ;
|
||||
0 [label="worst perimeter <= 106.05\ngini = 0.465\nsamples = 426\nvalue = [[269, 157]\n[157, 269]]", fillcolor="#e5813908"] ;
|
||||
1 [label="worst concave points <= 0.159\ngini = 0.067\nsamples = 259\nvalue = [[250, 9]\n[9, 250]]", fillcolor="#e58139db"] ;
|
||||
0 -> 1 [labeldistance=2.5, labelangle=45, headlabel="True"] ;
|
||||
2 [label="worst concave points <= 0.135\ngini = 0.031\nsamples = 253\nvalue = [[249, 4]\n[4, 249]]", fillcolor="#e58139ee"] ;
|
||||
1 -> 2 ;
|
||||
3 [label="area error <= 48.975\ngini = 0.008\nsamples = 242\nvalue = [[241, 1]\n[1, 241]]", fillcolor="#e58139fb"] ;
|
||||
2 -> 3 ;
|
||||
4 [label="gini = 0.0\nsamples = 239\nvalue = [[239, 0]\n[0, 239]]", fillcolor="#e58139ff"] ;
|
||||
3 -> 4 ;
|
||||
5 [label="symmetry error <= 0.025\ngini = 0.444\nsamples = 3\nvalue = [[2, 1]\n[1, 2]]", fillcolor="#e5813913"] ;
|
||||
3 -> 5 ;
|
||||
6 [label="gini = 0.0\nsamples = 1\nvalue = [[0, 1]\n[1, 0]]", fillcolor="#e58139ff"] ;
|
||||
5 -> 6 ;
|
||||
7 [label="gini = 0.0\nsamples = 2\nvalue = [[2, 0]\n[0, 2]]", fillcolor="#e58139ff"] ;
|
||||
5 -> 7 ;
|
||||
8 [label="mean texture <= 20.84\ngini = 0.397\nsamples = 11\nvalue = [[8, 3]\n[3, 8]]", fillcolor="#e581392c"] ;
|
||||
2 -> 8 ;
|
||||
9 [label="gini = 0.0\nsamples = 8\nvalue = [[8, 0]\n[0, 8]]", fillcolor="#e58139ff"] ;
|
||||
8 -> 9 ;
|
||||
10 [label="gini = 0.0\nsamples = 3\nvalue = [[0, 3]\n[3, 0]]", fillcolor="#e58139ff"] ;
|
||||
8 -> 10 ;
|
||||
11 [label="worst texture <= 24.785\ngini = 0.278\nsamples = 6\nvalue = [[1, 5]\n[5, 1]]", fillcolor="#e581396b"] ;
|
||||
1 -> 11 ;
|
||||
12 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
11 -> 12 ;
|
||||
13 [label="gini = 0.0\nsamples = 5\nvalue = [[0, 5]\n[5, 0]]", fillcolor="#e58139ff"] ;
|
||||
11 -> 13 ;
|
||||
14 [label="worst texture <= 20.645\ngini = 0.202\nsamples = 167\nvalue = [[19, 148]\n[148, 19]]", fillcolor="#e5813994"] ;
|
||||
0 -> 14 [labeldistance=2.5, labelangle=-45, headlabel="False"] ;
|
||||
15 [label="worst perimeter <= 116.8\ngini = 0.375\nsamples = 16\nvalue = [[12, 4]\n[4, 12]]", fillcolor="#e5813938"] ;
|
||||
14 -> 15 ;
|
||||
16 [label="gini = 0.0\nsamples = 11\nvalue = [[11, 0]\n[0, 11]]", fillcolor="#e58139ff"] ;
|
||||
15 -> 16 ;
|
||||
17 [label="mean concavity <= 0.06\ngini = 0.32\nsamples = 5\nvalue = [[1, 4]\n[4, 1]]", fillcolor="#e5813955"] ;
|
||||
15 -> 17 ;
|
||||
18 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
17 -> 18 ;
|
||||
19 [label="gini = 0.0\nsamples = 4\nvalue = [[0, 4]\n[4, 0]]", fillcolor="#e58139ff"] ;
|
||||
17 -> 19 ;
|
||||
20 [label="mean concave points <= 0.049\ngini = 0.088\nsamples = 151\nvalue = [[7, 144]\n[144, 7]]", fillcolor="#e58139d0"] ;
|
||||
14 -> 20 ;
|
||||
21 [label="concave points error <= 0.01\ngini = 0.48\nsamples = 15\nvalue = [[6, 9]\n[9, 6]]", fillcolor="#e5813900"] ;
|
||||
20 -> 21 ;
|
||||
22 [label="gini = 0.0\nsamples = 9\nvalue = [[0, 9]\n[9, 0]]", fillcolor="#e58139ff"] ;
|
||||
21 -> 22 ;
|
||||
23 [label="gini = 0.0\nsamples = 6\nvalue = [[6, 0]\n[0, 6]]", fillcolor="#e58139ff"] ;
|
||||
21 -> 23 ;
|
||||
24 [label="mean smoothness <= 0.079\ngini = 0.015\nsamples = 136\nvalue = [[1, 135]\n[135, 1]]", fillcolor="#e58139f7"] ;
|
||||
20 -> 24 ;
|
||||
25 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
24 -> 25 ;
|
||||
26 [label="gini = 0.0\nsamples = 135\nvalue = [[0, 135]\n[135, 0]]", fillcolor="#e58139ff"] ;
|
||||
24 -> 26 ;
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
# CART on the Bank Note dataset
|
||||
from random import seed
|
||||
from random import randrange
|
||||
from csv import reader
|
||||
|
||||
# Load a CSV file
|
||||
def load_csv(filename):
|
||||
file = open(filename, "rb")
|
||||
lines = reader(file)
|
||||
dataset = list(lines)
|
||||
return dataset
|
||||
|
||||
# Convert string column to float
|
||||
def str_column_to_float(dataset, column):
|
||||
for row in dataset:
|
||||
row[column] = float(row[column].strip())
|
||||
|
||||
# Split a dataset into k folds
|
||||
def cross_validation_split(dataset, n_folds):
|
||||
dataset_split = list()
|
||||
dataset_copy = list(dataset)
|
||||
fold_size = int(len(dataset) / n_folds)
|
||||
for i in range(n_folds):
|
||||
fold = list()
|
||||
while len(fold) < fold_size:
|
||||
index = randrange(len(dataset_copy))
|
||||
fold.append(dataset_copy.pop(index))
|
||||
dataset_split.append(fold)
|
||||
return dataset_split
|
||||
|
||||
# Calculate accuracy percentage
|
||||
def accuracy_metric(actual, predicted):
|
||||
correct = 0
|
||||
for i in range(len(actual)):
|
||||
if actual[i] == predicted[i]:
|
||||
correct += 1
|
||||
return correct / float(len(actual)) * 100.0
|
||||
|
||||
# Evaluate an algorithm using a cross validation split
|
||||
def evaluate_algorithm(dataset, algorithm, n_folds, *args):
|
||||
folds = cross_validation_split(dataset, n_folds)
|
||||
scores = list()
|
||||
for fold in folds:
|
||||
train_set = list(folds)
|
||||
train_set.remove(fold)
|
||||
train_set = sum(train_set, [])
|
||||
test_set = list()
|
||||
for row in fold:
|
||||
row_copy = list(row)
|
||||
test_set.append(row_copy)
|
||||
row_copy[-1] = None
|
||||
predicted = algorithm(train_set, test_set, *args)
|
||||
actual = [row[-1] for row in fold]
|
||||
accuracy = accuracy_metric(actual, predicted)
|
||||
scores.append(accuracy)
|
||||
return scores
|
||||
|
||||
# Split a dataset based on an attribute and an attribute value
|
||||
def test_split(index, value, dataset):
|
||||
left, right = list(), list()
|
||||
for row in dataset:
|
||||
if row[index] < value:
|
||||
left.append(row)
|
||||
else:
|
||||
right.append(row)
|
||||
return left, right
|
||||
|
||||
# Calculate the Gini index for a split dataset
|
||||
def gini_index(groups, classes):
|
||||
# count all samples at split point
|
||||
n_instances = float(sum([len(group) for group in groups]))
|
||||
# sum weighted Gini index for each group
|
||||
gini = 0.0
|
||||
for group in groups:
|
||||
size = float(len(group))
|
||||
# avoid divide by zero
|
||||
if size == 0:
|
||||
continue
|
||||
score = 0.0
|
||||
# score the group based on the score for each class
|
||||
for class_val in classes:
|
||||
p = [row[-1] for row in group].count(class_val) / size
|
||||
score += p * p
|
||||
# weight the group score by its relative size
|
||||
gini += (1.0 - score) * (size / n_instances)
|
||||
return gini
|
||||
|
||||
# Select the best split point for a dataset
|
||||
def get_split(dataset):
|
||||
class_values = list(set(row[-1] for row in dataset))
|
||||
b_index, b_value, b_score, b_groups = 999, 999, 999, None
|
||||
for index in range(len(dataset[0])-1):
|
||||
for row in dataset:
|
||||
groups = test_split(index, row[index], dataset)
|
||||
gini = gini_index(groups, class_values)
|
||||
if gini < b_score:
|
||||
b_index, b_value, b_score, b_groups = index, row[index], gini, groups
|
||||
return {'index':b_index, 'value':b_value, 'groups':b_groups}
|
||||
|
||||
# Create a terminal node value
|
||||
def to_terminal(group):
|
||||
outcomes = [row[-1] for row in group]
|
||||
return max(set(outcomes), key=outcomes.count)
|
||||
|
||||
# Create child splits for a node or make terminal
|
||||
def split(node, max_depth, min_size, depth):
|
||||
left, right = node['groups']
|
||||
del(node['groups'])
|
||||
# check for a no split
|
||||
if not left or not right:
|
||||
node['left'] = node['right'] = to_terminal(left + right)
|
||||
return
|
||||
# check for max depth
|
||||
if depth >= max_depth:
|
||||
node['left'], node['right'] = to_terminal(left), to_terminal(right)
|
||||
return
|
||||
# process left child
|
||||
if len(left) <= min_size:
|
||||
node['left'] = to_terminal(left)
|
||||
else:
|
||||
node['left'] = get_split(left)
|
||||
split(node['left'], max_depth, min_size, depth+1)
|
||||
# process right child
|
||||
if len(right) <= min_size:
|
||||
node['right'] = to_terminal(right)
|
||||
else:
|
||||
node['right'] = get_split(right)
|
||||
split(node['right'], max_depth, min_size, depth+1)
|
||||
|
||||
# Build a decision tree
|
||||
def build_tree(train, max_depth, min_size):
|
||||
root = get_split(train)
|
||||
split(root, max_depth, min_size, 1)
|
||||
return root
|
||||
|
||||
# Make a prediction with a decision tree
|
||||
def predict(node, row):
|
||||
if row[node['index']] < node['value']:
|
||||
if isinstance(node['left'], dict):
|
||||
return predict(node['left'], row)
|
||||
else:
|
||||
return node['left']
|
||||
else:
|
||||
if isinstance(node['right'], dict):
|
||||
return predict(node['right'], row)
|
||||
else:
|
||||
return node['right']
|
||||
|
||||
# Classification and Regression Tree Algorithm
|
||||
def decision_tree(train, test, max_depth, min_size):
|
||||
tree = build_tree(train, max_depth, min_size)
|
||||
predictions = list()
|
||||
for row in test:
|
||||
prediction = predict(tree, row)
|
||||
predictions.append(prediction)
|
||||
return(predictions)
|
||||
|
||||
# Test CART on Bank Note dataset
|
||||
seed(1)
|
||||
# load and prepare data
|
||||
filename = 'DataFiles/rideclass.csv'
|
||||
dataset = load_csv(filename)
|
||||
# convert string attributes to integers
|
||||
for i in range(len(dataset[0])):
|
||||
str_column_to_float(dataset, i)
|
||||
# evaluate algorithm
|
||||
n_folds = 5
|
||||
max_depth = 5
|
||||
min_size = 10
|
||||
scores = evaluate_algorithm(dataset, decision_tree, n_folds, max_depth, min_size)
|
||||
print('Scores: %s' % scores)
|
||||
print('Mean Accuracy: %.3f%%' % (sum(scores)/float(len(scores))))
|
||||
@@ -0,0 +1,172 @@
|
||||
# CART on the Bank Note dataset
|
||||
from random import seed
|
||||
from random import randrange
|
||||
from csv import reader
|
||||
|
||||
# Load a CSV file
|
||||
def load_csv(filename):
|
||||
file = open(filename, "rb")
|
||||
lines = reader(file)
|
||||
dataset = list(lines)
|
||||
return dataset
|
||||
|
||||
# Convert string column to float
|
||||
def str_column_to_float(dataset, column):
|
||||
for row in dataset:
|
||||
row[column] = float(row[column].strip())
|
||||
|
||||
# Split a dataset into k folds
|
||||
def cross_validation_split(dataset, n_folds):
|
||||
dataset_split = list()
|
||||
dataset_copy = list(dataset)
|
||||
fold_size = int(len(dataset) / n_folds)
|
||||
for i in range(n_folds):
|
||||
fold = list()
|
||||
while len(fold) < fold_size:
|
||||
index = randrange(len(dataset_copy))
|
||||
fold.append(dataset_copy.pop(index))
|
||||
dataset_split.append(fold)
|
||||
return dataset_split
|
||||
|
||||
# Calculate accuracy percentage
|
||||
def accuracy_metric(actual, predicted):
|
||||
correct = 0
|
||||
for i in range(len(actual)):
|
||||
if actual[i] == predicted[i]:
|
||||
correct += 1
|
||||
return correct / float(len(actual)) * 100.0
|
||||
|
||||
# Evaluate an algorithm using a cross validation split
|
||||
def evaluate_algorithm(dataset, algorithm, n_folds, *args):
|
||||
folds = cross_validation_split(dataset, n_folds)
|
||||
scores = list()
|
||||
for fold in folds:
|
||||
train_set = list(folds)
|
||||
train_set.remove(fold)
|
||||
train_set = sum(train_set, [])
|
||||
test_set = list()
|
||||
for row in fold:
|
||||
row_copy = list(row)
|
||||
test_set.append(row_copy)
|
||||
row_copy[-1] = None
|
||||
predicted = algorithm(train_set, test_set, *args)
|
||||
actual = [row[-1] for row in fold]
|
||||
accuracy = accuracy_metric(actual, predicted)
|
||||
scores.append(accuracy)
|
||||
return scores
|
||||
|
||||
# Split a dataset based on an attribute and an attribute value
|
||||
def test_split(index, value, dataset):
|
||||
left, right = list(), list()
|
||||
for row in dataset:
|
||||
if row[index] < value:
|
||||
left.append(row)
|
||||
else:
|
||||
right.append(row)
|
||||
return left, right
|
||||
|
||||
# Calculate the Gini index for a split dataset
|
||||
def gini_index(groups, classes):
|
||||
# count all samples at split point
|
||||
n_instances = float(sum([len(group) for group in groups]))
|
||||
# sum weighted Gini index for each group
|
||||
gini = 0.0
|
||||
for group in groups:
|
||||
size = float(len(group))
|
||||
# avoid divide by zero
|
||||
if size == 0:
|
||||
continue
|
||||
score = 0.0
|
||||
# score the group based on the score for each class
|
||||
for class_val in classes:
|
||||
p = [row[-1] for row in group].count(class_val) / size
|
||||
score += p * p
|
||||
# weight the group score by its relative size
|
||||
gini += (1.0 - score) * (size / n_instances)
|
||||
return gini
|
||||
|
||||
# Select the best split point for a dataset
|
||||
def get_split(dataset):
|
||||
class_values = list(set(row[-1] for row in dataset))
|
||||
b_index, b_value, b_score, b_groups = 999, 999, 999, None
|
||||
for index in range(len(dataset[0])-1):
|
||||
for row in dataset:
|
||||
groups = test_split(index, row[index], dataset)
|
||||
gini = gini_index(groups, class_values)
|
||||
if gini < b_score:
|
||||
b_index, b_value, b_score, b_groups = index, row[index], gini, groups
|
||||
return {'index':b_index, 'value':b_value, 'groups':b_groups}
|
||||
|
||||
# Create a terminal node value
|
||||
def to_terminal(group):
|
||||
outcomes = [row[-1] for row in group]
|
||||
return max(set(outcomes), key=outcomes.count)
|
||||
|
||||
# Create child splits for a node or make terminal
|
||||
def split(node, max_depth, min_size, depth):
|
||||
left, right = node['groups']
|
||||
del(node['groups'])
|
||||
# check for a no split
|
||||
if not left or not right:
|
||||
node['left'] = node['right'] = to_terminal(left + right)
|
||||
return
|
||||
# check for max depth
|
||||
if depth >= max_depth:
|
||||
node['left'], node['right'] = to_terminal(left), to_terminal(right)
|
||||
return
|
||||
# process left child
|
||||
if len(left) <= min_size:
|
||||
node['left'] = to_terminal(left)
|
||||
else:
|
||||
node['left'] = get_split(left)
|
||||
split(node['left'], max_depth, min_size, depth+1)
|
||||
# process right child
|
||||
if len(right) <= min_size:
|
||||
node['right'] = to_terminal(right)
|
||||
else:
|
||||
node['right'] = get_split(right)
|
||||
split(node['right'], max_depth, min_size, depth+1)
|
||||
|
||||
# Build a decision tree
|
||||
def build_tree(train, max_depth, min_size):
|
||||
root = get_split(train)
|
||||
split(root, max_depth, min_size, 1)
|
||||
return root
|
||||
|
||||
# Make a prediction with a decision tree
|
||||
def predict(node, row):
|
||||
if row[node['index']] < node['value']:
|
||||
if isinstance(node['left'], dict):
|
||||
return predict(node['left'], row)
|
||||
else:
|
||||
return node['left']
|
||||
else:
|
||||
if isinstance(node['right'], dict):
|
||||
return predict(node['right'], row)
|
||||
else:
|
||||
return node['right']
|
||||
|
||||
# Classification and Regression Tree Algorithm
|
||||
def decision_tree(train, test, max_depth, min_size):
|
||||
tree = build_tree(train, max_depth, min_size)
|
||||
predictions = list()
|
||||
for row in test:
|
||||
prediction = predict(tree, row)
|
||||
predictions.append(prediction)
|
||||
return(predictions)
|
||||
|
||||
# Test CART on Bank Note dataset
|
||||
seed(1)
|
||||
# load and prepare data
|
||||
filename = 'DataFiles/ride.csv'
|
||||
dataset = load_csv(filename)
|
||||
# convert string attributes to integers
|
||||
for i in range(len(dataset[0])):
|
||||
str_column_to_float(dataset, i)
|
||||
# evaluate algorithm
|
||||
n_folds = 5
|
||||
max_depth = 5
|
||||
min_size = 10
|
||||
scores = evaluate_algorithm(dataset, decision_tree, n_folds, max_depth, min_size)
|
||||
print('Scores: %s' % scores)
|
||||
print('Mean Accuracy: %.3f%%' % (sum(scores)/float(len(scores))))
|
||||
@@ -0,0 +1,187 @@
|
||||
import re
|
||||
import math
|
||||
from collections import deque
|
||||
|
||||
# x is examples in training set
|
||||
# y is set of attributes
|
||||
# label is target attributes
|
||||
# Node is a class which has properties values, childs, and next
|
||||
# root is top node in the decision tree
|
||||
|
||||
class Node(object):
|
||||
def __init__(self):
|
||||
self.value = None
|
||||
self.next = None
|
||||
self.childs = None
|
||||
|
||||
# Simple class of Decision Tree
|
||||
# Aimed for who want to learn Decision Tree, so it is not optimized
|
||||
class DecisionTree(object):
|
||||
def __init__(self, sample, attributes, labels):
|
||||
self.sample = sample
|
||||
self.attributes = attributes
|
||||
self.labels = labels
|
||||
self.labelCodes = None
|
||||
self.labelCodesCount = None
|
||||
self.initLabelCodes()
|
||||
# print(self.labelCodes)
|
||||
self.root = None
|
||||
self.entropy = self.getEntropy([x for x in range(len(self.labels))])
|
||||
|
||||
def initLabelCodes(self):
|
||||
self.labelCodes = []
|
||||
self.labelCodesCount = []
|
||||
for l in self.labels:
|
||||
if l not in self.labelCodes:
|
||||
self.labelCodes.append(l)
|
||||
self.labelCodesCount.append(0)
|
||||
self.labelCodesCount[self.labelCodes.index(l)] += 1
|
||||
|
||||
def getLabelCodeId(self, sampleId):
|
||||
return self.labelCodes.index(self.labels[sampleId])
|
||||
|
||||
def getAttributeValues(self, sampleIds, attributeId):
|
||||
vals = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in vals:
|
||||
vals.append(val)
|
||||
# print(vals)
|
||||
return vals
|
||||
|
||||
def getEntropy(self, sampleIds):
|
||||
entropy = 0
|
||||
labelCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCount[self.getLabelCodeId(sid)] += 1
|
||||
# print("-ge", labelCount)
|
||||
for lv in labelCount:
|
||||
# print(lv)
|
||||
if lv != 0:
|
||||
entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
|
||||
else:
|
||||
entropy += 0
|
||||
return entropy
|
||||
|
||||
def getDominantLabel(self, sampleIds):
|
||||
labelCodesCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
|
||||
return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
|
||||
|
||||
def getInformationGain(self, sampleIds, attributeId):
|
||||
gain = self.getEntropy(sampleIds)
|
||||
attributeVals = []
|
||||
attributeValsCount = []
|
||||
attributeValsIds = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in attributeVals:
|
||||
attributeVals.append(val)
|
||||
attributeValsCount.append(0)
|
||||
attributeValsIds.append([])
|
||||
vid = attributeVals.index(val)
|
||||
attributeValsCount[vid] += 1
|
||||
attributeValsIds[vid].append(sid)
|
||||
# print("-gig", self.attributes[attributeId])
|
||||
for vc, vids in zip(attributeValsCount, attributeValsIds):
|
||||
# print("-gig", vids)
|
||||
gain -= vc/len(sampleIds) * self.getEntropy(vids)
|
||||
return gain
|
||||
|
||||
def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
|
||||
attributesEntropy = [0] * len(attributeIds)
|
||||
for i, attId in zip(range(len(attributeIds)), attributeIds):
|
||||
attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
|
||||
maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
|
||||
return self.attributes[maxId], maxId
|
||||
|
||||
def isSingleLabeled(self, sampleIds):
|
||||
label = self.labels[sampleIds[0]]
|
||||
for sid in sampleIds:
|
||||
if self.labels[sid] != label:
|
||||
return False
|
||||
return True
|
||||
|
||||
def getLabel(self, sampleId):
|
||||
return self.labels[sampleId]
|
||||
|
||||
def id3(self):
|
||||
sampleIds = [x for x in range(len(self.sample))]
|
||||
attributeIds = [x for x in range(len(self.attributes))]
|
||||
self.root = self.id3Recv(sampleIds, attributeIds, self.root)
|
||||
|
||||
def id3Recv(self, sampleIds, attributeIds, root):
|
||||
root = Node() # Initialize current root
|
||||
if self.isSingleLabeled(sampleIds):
|
||||
root.value = self.labels[sampleIds[0]]
|
||||
return root
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) == 0:
|
||||
root.value = self.getDominantLabel(sampleIds)
|
||||
return root
|
||||
bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
|
||||
sampleIds, attributeIds)
|
||||
# print(bestAttrName)
|
||||
root.value = bestAttrName
|
||||
root.childs = [] # Create list of children
|
||||
for value in self.getAttributeValues(sampleIds, bestAttrId):
|
||||
# print(value)
|
||||
child = Node()
|
||||
child.value = value
|
||||
root.childs.append(child) # Append new child node to current
|
||||
# root
|
||||
childSampleIds = []
|
||||
for sid in sampleIds:
|
||||
if self.sample[sid][bestAttrId] == value:
|
||||
childSampleIds.append(sid)
|
||||
if len(childSampleIds) == 0:
|
||||
child.next = self.getDominantLabel(sampleIds)
|
||||
else:
|
||||
# print(bestAttrName, bestAttrId)
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) > 0 and bestAttrId in attributeIds:
|
||||
toRemove = attributeIds.index(bestAttrId)
|
||||
attributeIds.pop(toRemove)
|
||||
child.next = self.id3Recv(
|
||||
childSampleIds, attributeIds, child.next)
|
||||
return root
|
||||
|
||||
def printTree(self):
|
||||
if self.root:
|
||||
roots = deque()
|
||||
roots.append(self.root)
|
||||
while len(roots) > 0:
|
||||
root = roots.popleft()
|
||||
print(root.value)
|
||||
if root.childs:
|
||||
for child in root.childs:
|
||||
print('({})'.format(child.value))
|
||||
roots.append(child.next)
|
||||
elif root.next:
|
||||
print(root.next)
|
||||
|
||||
|
||||
def test():
|
||||
f = open('DataFiles/rideclass.csv')
|
||||
attributes = f.readline().split(',')
|
||||
attributes = attributes[1:len(attributes)-1]
|
||||
print(attributes)
|
||||
sample = f.readlines()
|
||||
f.close()
|
||||
for i in range(len(sample)):
|
||||
sample[i] = re.sub('\d+,', '', sample[i])
|
||||
sample[i] = sample[i].strip().split(',')
|
||||
labels = []
|
||||
for s in sample:
|
||||
labels.append(s.pop())
|
||||
# print(sample)
|
||||
# print(labels)
|
||||
decisionTree = DecisionTree(sample, attributes, labels)
|
||||
print("System entropy {}".format(decisionTree.entropy))
|
||||
decisionTree.id3()
|
||||
decisionTree.printTree()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
@@ -0,0 +1,187 @@
|
||||
import re
|
||||
import math
|
||||
from collections import deque
|
||||
|
||||
# x is examples in training set
|
||||
# y is set of attributes
|
||||
# label is target attributes
|
||||
# Node is a class which has properties values, childs, and next
|
||||
# root is top node in the decision tree
|
||||
|
||||
class Node(object):
|
||||
def __init__(self):
|
||||
self.value = None
|
||||
self.next = None
|
||||
self.childs = None
|
||||
|
||||
# Simple class of Decision Tree
|
||||
# Aimed for who want to learn Decision Tree, so it is not optimized
|
||||
class DecisionTree(object):
|
||||
def __init__(self, sample, attributes, labels):
|
||||
self.sample = sample
|
||||
self.attributes = attributes
|
||||
self.labels = labels
|
||||
self.labelCodes = None
|
||||
self.labelCodesCount = None
|
||||
self.initLabelCodes()
|
||||
# print(self.labelCodes)
|
||||
self.root = None
|
||||
self.entropy = self.getEntropy([x for x in range(len(self.labels))])
|
||||
|
||||
def initLabelCodes(self):
|
||||
self.labelCodes = []
|
||||
self.labelCodesCount = []
|
||||
for l in self.labels:
|
||||
if l not in self.labelCodes:
|
||||
self.labelCodes.append(l)
|
||||
self.labelCodesCount.append(0)
|
||||
self.labelCodesCount[self.labelCodes.index(l)] += 1
|
||||
|
||||
def getLabelCodeId(self, sampleId):
|
||||
return self.labelCodes.index(self.labels[sampleId])
|
||||
|
||||
def getAttributeValues(self, sampleIds, attributeId):
|
||||
vals = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in vals:
|
||||
vals.append(val)
|
||||
# print(vals)
|
||||
return vals
|
||||
|
||||
def getEntropy(self, sampleIds):
|
||||
entropy = 0
|
||||
labelCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCount[self.getLabelCodeId(sid)] += 1
|
||||
# print("-ge", labelCount)
|
||||
for lv in labelCount:
|
||||
# print(lv)
|
||||
if lv != 0:
|
||||
entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
|
||||
else:
|
||||
entropy += 0
|
||||
return entropy
|
||||
|
||||
def getDominantLabel(self, sampleIds):
|
||||
labelCodesCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
|
||||
return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
|
||||
|
||||
def getInformationGain(self, sampleIds, attributeId):
|
||||
gain = self.getEntropy(sampleIds)
|
||||
attributeVals = []
|
||||
attributeValsCount = []
|
||||
attributeValsIds = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in attributeVals:
|
||||
attributeVals.append(val)
|
||||
attributeValsCount.append(0)
|
||||
attributeValsIds.append([])
|
||||
vid = attributeVals.index(val)
|
||||
attributeValsCount[vid] += 1
|
||||
attributeValsIds[vid].append(sid)
|
||||
# print("-gig", self.attributes[attributeId])
|
||||
for vc, vids in zip(attributeValsCount, attributeValsIds):
|
||||
# print("-gig", vids)
|
||||
gain -= vc/len(sampleIds) * self.getEntropy(vids)
|
||||
return gain
|
||||
|
||||
def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
|
||||
attributesEntropy = [0] * len(attributeIds)
|
||||
for i, attId in zip(range(len(attributeIds)), attributeIds):
|
||||
attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
|
||||
maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
|
||||
return self.attributes[maxId], maxId
|
||||
|
||||
def isSingleLabeled(self, sampleIds):
|
||||
label = self.labels[sampleIds[0]]
|
||||
for sid in sampleIds:
|
||||
if self.labels[sid] != label:
|
||||
return False
|
||||
return True
|
||||
|
||||
def getLabel(self, sampleId):
|
||||
return self.labels[sampleId]
|
||||
|
||||
def id3(self):
|
||||
sampleIds = [x for x in range(len(self.sample))]
|
||||
attributeIds = [x for x in range(len(self.attributes))]
|
||||
self.root = self.id3Recv(sampleIds, attributeIds, self.root)
|
||||
|
||||
def id3Recv(self, sampleIds, attributeIds, root):
|
||||
root = Node() # Initialize current root
|
||||
if self.isSingleLabeled(sampleIds):
|
||||
root.value = self.labels[sampleIds[0]]
|
||||
return root
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) == 0:
|
||||
root.value = self.getDominantLabel(sampleIds)
|
||||
return root
|
||||
bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
|
||||
sampleIds, attributeIds)
|
||||
# print(bestAttrName)
|
||||
root.value = bestAttrName
|
||||
root.childs = [] # Create list of children
|
||||
for value in self.getAttributeValues(sampleIds, bestAttrId):
|
||||
# print(value)
|
||||
child = Node()
|
||||
child.value = value
|
||||
root.childs.append(child) # Append new child node to current
|
||||
# root
|
||||
childSampleIds = []
|
||||
for sid in sampleIds:
|
||||
if self.sample[sid][bestAttrId] == value:
|
||||
childSampleIds.append(sid)
|
||||
if len(childSampleIds) == 0:
|
||||
child.next = self.getDominantLabel(sampleIds)
|
||||
else:
|
||||
# print(bestAttrName, bestAttrId)
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) > 0 and bestAttrId in attributeIds:
|
||||
toRemove = attributeIds.index(bestAttrId)
|
||||
attributeIds.pop(toRemove)
|
||||
child.next = self.id3Recv(
|
||||
childSampleIds, attributeIds, child.next)
|
||||
return root
|
||||
|
||||
def printTree(self):
|
||||
if self.root:
|
||||
roots = deque()
|
||||
roots.append(self.root)
|
||||
while len(roots) > 0:
|
||||
root = roots.popleft()
|
||||
print(root.value)
|
||||
if root.childs:
|
||||
for child in root.childs:
|
||||
print('({})'.format(child.value))
|
||||
roots.append(child.next)
|
||||
elif root.next:
|
||||
print(root.next)
|
||||
|
||||
|
||||
def test():
|
||||
f = open('rideclass.csv')
|
||||
attributes = f.readline().split(',')
|
||||
attributes = attributes[1:len(attributes)-1]
|
||||
print(attributes)
|
||||
sample = f.readlines()
|
||||
f.close()
|
||||
for i in range(len(sample)):
|
||||
sample[i] = re.sub('\d+,', '', sample[i])
|
||||
sample[i] = sample[i].strip().split(',')
|
||||
labels = []
|
||||
for s in sample:
|
||||
labels.append(s.pop())
|
||||
# print(sample)
|
||||
# print(labels)
|
||||
decisionTree = DecisionTree(sample, attributes, labels)
|
||||
print("System entropy {}".format(decisionTree.entropy))
|
||||
decisionTree.id3()
|
||||
decisionTree.printTree()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
@@ -0,0 +1,30 @@
|
||||
from sklearn.datasets import load_breast_cancer
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.metrics import confusion_matrix
|
||||
from sklearn.tree import export_graphviz
|
||||
|
||||
from IPython.display import Image
|
||||
from pydot import graph_from_dot_data
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
cancer = load_breast_cancer()
|
||||
X = pd.DataFrame(cancer.data, columns=cancer.feature_names)
|
||||
print(X)
|
||||
y = pd.Categorical.from_codes(cancer.target, cancer.target_names)
|
||||
y = pd.get_dummies(y)
|
||||
print(y)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)
|
||||
tree_clf = DecisionTreeClassifier(max_depth=5)
|
||||
tree_clf.fit(X_train, y_train)
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file="cancer.dot",
|
||||
feature_names=cancer.feature_names,
|
||||
class_names=cancer.target_names,
|
||||
rounded=True,
|
||||
filled=True
|
||||
)
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
from sklearn.datasets import load_breast_cancer
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.metrics import confusion_matrix
|
||||
from sklearn.tree import export_graphviz
|
||||
|
||||
from IPython.display import Image
|
||||
from pydot import graph_from_dot_data
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
cancer = load_breast_cancer()
|
||||
X = pd.DataFrame(cancer.data, columns=cancer.feature_names)
|
||||
y = pd.Categorical.from_codes(cancer.target, cancer.target_names)
|
||||
y = pd.get_dummies(y)
|
||||
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)
|
||||
tree_clf = DecisionTreeClassifier(max_depth=5)
|
||||
tree_clf.fit(X_train, y_train)
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file="cancer.dot",
|
||||
feature_names=cancer.feature_names,
|
||||
class_names=cancer.target_names,
|
||||
rounded=True,
|
||||
filled=True
|
||||
)
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
# Split a dataset based on an attribute and an attribute value
|
||||
def test_split(index, value, dataset):
|
||||
left, right = list(), list()
|
||||
for row in dataset:
|
||||
if row[index] < value:
|
||||
left.append(row)
|
||||
else:
|
||||
right.append(row)
|
||||
return left, right
|
||||
|
||||
# Calculate the Gini index for a split dataset
|
||||
def gini_index(groups, classes):
|
||||
# count all samples at split point
|
||||
n_instances = float(sum([len(group) for group in groups]))
|
||||
# sum weighted Gini index for each group
|
||||
gini = 0.0
|
||||
for group in groups:
|
||||
size = float(len(group))
|
||||
# avoid divide by zero
|
||||
if size == 0:
|
||||
continue
|
||||
score = 0.0
|
||||
# score the group based on the score for each class
|
||||
for class_val in classes:
|
||||
p = [row[-1] for row in group].count(class_val) / size
|
||||
score += p * p
|
||||
# weight the group score by its relative size
|
||||
gini += (1.0 - score) * (size / n_instances)
|
||||
return gini
|
||||
|
||||
# Select the best split point for a dataset
|
||||
def get_split(dataset):
|
||||
class_values = list(set(row[-1] for row in dataset))
|
||||
b_index, b_value, b_score, b_groups = 999, 999, 999, None
|
||||
for index in range(len(dataset[0])-1):
|
||||
for row in dataset:
|
||||
groups = test_split(index, row[index], dataset)
|
||||
gini = gini_index(groups, class_values)
|
||||
print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini))
|
||||
if gini < b_score:
|
||||
b_index, b_value, b_score, b_groups = index, row[index], gini, groups
|
||||
return {'index':b_index, 'value':b_value, 'groups':b_groups}
|
||||
|
||||
dataset = [[2.771244718,1.784783929,0],
|
||||
[1.728571309,1.169761413,0],
|
||||
[3.678319846,2.81281357,0],
|
||||
[3.961043357,2.61995032,0],
|
||||
[2.999208922,2.209014212,0],
|
||||
[7.497545867,3.162953546,1],
|
||||
[9.00220326,3.339047188,1],
|
||||
[7.444542326,0.476683375,1],
|
||||
[10.12493903,3.234550982,1],
|
||||
[6.642287351,3.319983761,1]]
|
||||
split = get_split(dataset)
|
||||
print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
|
||||
@@ -0,0 +1,28 @@
|
||||
# Common imports
|
||||
import numpy as np
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.datasets import make_moons
|
||||
from sklearn.tree import export_graphviz
|
||||
from pydot import graph_from_dot_data
|
||||
import pandas as pd
|
||||
|
||||
|
||||
np.random.seed(42)
|
||||
X, y = make_moons(n_samples=100, noise=0.25, random_state=53)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)
|
||||
tree_clf = DecisionTreeClassifier(max_depth=5)
|
||||
tree_clf.fit(X_train, y_train)
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file="moons.dot",
|
||||
# feature_names=tree_clf.feature_names,
|
||||
# class_names=tree_clf.target_names,
|
||||
rounded=True,
|
||||
filled=True
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
# Common imports
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.tree import export_graphviz
|
||||
from sklearn.preprocessing import StandardScaler, OneHotEncoder
|
||||
from sklearn.compose import ColumnTransformer
|
||||
from IPython.display import Image
|
||||
from pydot import graph_from_dot_data
|
||||
import os
|
||||
|
||||
# Where to save the figures and data files
|
||||
PROJECT_ROOT_DIR = "Results"
|
||||
FIGURE_ID = "Results/FigureFiles"
|
||||
DATA_ID = "DataFiles/"
|
||||
|
||||
if not os.path.exists(PROJECT_ROOT_DIR):
|
||||
os.mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
if not os.path.exists(FIGURE_ID):
|
||||
os.makedirs(FIGURE_ID)
|
||||
|
||||
if not os.path.exists(DATA_ID):
|
||||
os.makedirs(DATA_ID)
|
||||
|
||||
def image_path(fig_id):
|
||||
return os.path.join(FIGURE_ID, fig_id)
|
||||
|
||||
def data_path(dat_id):
|
||||
return os.path.join(DATA_ID, dat_id)
|
||||
|
||||
def save_fig(fig_id):
|
||||
plt.savefig(image_path(fig_id) + ".png", format='png')
|
||||
|
||||
infile = open(data_path("ride.csv"),'r')
|
||||
|
||||
# Read the experimental data with Pandas
|
||||
from IPython.display import display
|
||||
ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
|
||||
ridedata = pd.DataFrame(ridedata)
|
||||
display(ridedata)
|
||||
# Features and targets
|
||||
X = ridedata.loc[:, ridedata.columns != 'Ride'].values
|
||||
display(X)
|
||||
y = ridedata.loc[:, ridedata.columns == 'Ride'].values
|
||||
display(y)
|
||||
# Categorical variables to one-hot's
|
||||
onehotencoder = OneHotEncoder(categories="auto")
|
||||
|
||||
X = ColumnTransformer([("", onehotencoder)]).fit_transform(X)
|
||||
y.shape
|
||||
|
||||
display(X)
|
||||
display(y)
|
||||
|
||||
|
||||
"""
|
||||
X = pd.DataFrame(ridedata.data, columns=ridedata.feature_names)
|
||||
y = pd.Categorical.from_codes(ridedata.target, ridedata.target_names)
|
||||
y = pd.get_dummies(y)
|
||||
|
||||
|
||||
tree_clf = DecisionTreeClassifier(max_depth=2)
|
||||
tree_clf.fit(X, y)
|
||||
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file="ride.dot",
|
||||
feature_names=tree_clf.feature_names,
|
||||
class_names=tree_clf.target_names,
|
||||
rounded=True,
|
||||
filled=True
|
||||
)
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
# Common imports
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.tree import export_graphviz
|
||||
from sklearn.preprocessing import StandardScaler, OneHotEncoder
|
||||
from sklearn.compose import ColumnTransformer
|
||||
from IPython.display import Image
|
||||
from pydot import graph_from_dot_data
|
||||
import os
|
||||
|
||||
# Where to save the figures and data files
|
||||
PROJECT_ROOT_DIR = "Results"
|
||||
FIGURE_ID = "Results/FigureFiles"
|
||||
DATA_ID = "DataFiles/"
|
||||
|
||||
if not os.path.exists(PROJECT_ROOT_DIR):
|
||||
os.mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
if not os.path.exists(FIGURE_ID):
|
||||
os.makedirs(FIGURE_ID)
|
||||
|
||||
if not os.path.exists(DATA_ID):
|
||||
os.makedirs(DATA_ID)
|
||||
|
||||
def image_path(fig_id):
|
||||
return os.path.join(FIGURE_ID, fig_id)
|
||||
|
||||
def data_path(dat_id):
|
||||
return os.path.join(DATA_ID, dat_id)
|
||||
|
||||
def save_fig(fig_id):
|
||||
plt.savefig(image_path(fig_id) + ".png", format='png')
|
||||
|
||||
infile = open(data_path("ride.csv"),'r')
|
||||
|
||||
# Read the experimental data with Pandas
|
||||
from IPython.display import display
|
||||
ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
|
||||
ridedata = pd.DataFrame(ridedata)
|
||||
display(ridedata)
|
||||
# Features and targets
|
||||
X = ridedata.loc[:, ridedata.columns != 'Ride'].values
|
||||
display(X)
|
||||
y = ridedata.loc[:, ridedata.columns == 'Ride'].values
|
||||
display(y)
|
||||
# Categorical variables to one-hot's
|
||||
onehotencoder = OneHotEncoder(categories="auto")
|
||||
|
||||
X = ColumnTransformer(
|
||||
[("", onehotencoder)],
|
||||
remainder="passthrough").fit_transform(X)
|
||||
y.shape
|
||||
|
||||
display(X)
|
||||
display(y)
|
||||
|
||||
|
||||
"""
|
||||
X = pd.DataFrame(ridedata.data, columns=ridedata.feature_names)
|
||||
y = pd.Categorical.from_codes(ridedata.target, ridedata.target_names)
|
||||
y = pd.get_dummies(y)
|
||||
|
||||
|
||||
tree_clf = DecisionTreeClassifier(max_depth=2)
|
||||
tree_clf.fit(X, y)
|
||||
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file="ride.dot",
|
||||
feature_names=tree_clf.feature_names,
|
||||
class_names=tree_clf.target_names,
|
||||
rounded=True,
|
||||
filled=True
|
||||
)
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user