added codes
This commit is contained in:
Morten Hjorth-Jensen
@@ -0,0 +1,27 @@
|
||||
import numpy as np
|
||||
class AdaGrad:
|
||||
def __init__(self, learning_rate=0.01, epsilon=1e-8):
|
||||
self.learning_rate = learning_rate
|
||||
self.epsilon = epsilon
|
||||
self.gradient_squared = None
|
||||
def update(self, weights, gradient):
|
||||
if self.gradient_squared is None:
|
||||
self.gradient_squared = np.zeros_like(weights)
|
||||
# Accumulate squared gradients
|
||||
self.gradient_squared += gradient ** 2
|
||||
|
||||
# Update weights
|
||||
adjusted_grads = gradient / (np.sqrt(self.gradient_squared) + self.epsilon)
|
||||
weights -= self.learning_rate * adjusted_grads
|
||||
|
||||
return weights
|
||||
# Example usage
|
||||
if __name__ == "__main__":
|
||||
# Sample data and gradient
|
||||
np.random.seed(0)
|
||||
weights = np.random.rand(3)
|
||||
gradients = np.random.rand(100, 3) # Simulating 100 gradients
|
||||
optimizer = AdaGrad(learning_rate=0.1)
|
||||
for grad in gradients:
|
||||
weights = optimizer.update(weights, grad)
|
||||
print("Updated weights:", weights)
|
||||
@@ -1,187 +1,81 @@
|
||||
import re
|
||||
import math
|
||||
from collections import deque
|
||||
|
||||
# x is examples in training set
|
||||
# y is set of attributes
|
||||
# label is target attributes
|
||||
# Node is a class which has properties values, childs, and next
|
||||
# root is top node in the decision tree
|
||||
|
||||
class Node(object):
|
||||
def __init__(self):
|
||||
self.value = None
|
||||
self.next = None
|
||||
self.childs = None
|
||||
|
||||
# Simple class of Decision Tree
|
||||
# Aimed for who want to learn Decision Tree, so it is not optimized
|
||||
class DecisionTree(object):
|
||||
def __init__(self, sample, attributes, labels):
|
||||
self.sample = sample
|
||||
self.attributes = attributes
|
||||
self.labels = labels
|
||||
self.labelCodes = None
|
||||
self.labelCodesCount = None
|
||||
self.initLabelCodes()
|
||||
# print(self.labelCodes)
|
||||
self.root = None
|
||||
self.entropy = self.getEntropy([x for x in range(len(self.labels))])
|
||||
|
||||
def initLabelCodes(self):
|
||||
self.labelCodes = []
|
||||
self.labelCodesCount = []
|
||||
for l in self.labels:
|
||||
if l not in self.labelCodes:
|
||||
self.labelCodes.append(l)
|
||||
self.labelCodesCount.append(0)
|
||||
self.labelCodesCount[self.labelCodes.index(l)] += 1
|
||||
|
||||
def getLabelCodeId(self, sampleId):
|
||||
return self.labelCodes.index(self.labels[sampleId])
|
||||
|
||||
def getAttributeValues(self, sampleIds, attributeId):
|
||||
vals = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in vals:
|
||||
vals.append(val)
|
||||
# print(vals)
|
||||
return vals
|
||||
|
||||
def getEntropy(self, sampleIds):
|
||||
entropy = 0
|
||||
labelCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCount[self.getLabelCodeId(sid)] += 1
|
||||
# print("-ge", labelCount)
|
||||
for lv in labelCount:
|
||||
# print(lv)
|
||||
if lv != 0:
|
||||
entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2)
|
||||
else:
|
||||
entropy += 0
|
||||
return entropy
|
||||
|
||||
def getDominantLabel(self, sampleIds):
|
||||
labelCodesCount = [0] * len(self.labelCodes)
|
||||
for sid in sampleIds:
|
||||
labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1
|
||||
return self.labelCodes[labelCodesCount.index(max(labelCodesCount))]
|
||||
|
||||
def getInformationGain(self, sampleIds, attributeId):
|
||||
gain = self.getEntropy(sampleIds)
|
||||
attributeVals = []
|
||||
attributeValsCount = []
|
||||
attributeValsIds = []
|
||||
for sid in sampleIds:
|
||||
val = self.sample[sid][attributeId]
|
||||
if val not in attributeVals:
|
||||
attributeVals.append(val)
|
||||
attributeValsCount.append(0)
|
||||
attributeValsIds.append([])
|
||||
vid = attributeVals.index(val)
|
||||
attributeValsCount[vid] += 1
|
||||
attributeValsIds[vid].append(sid)
|
||||
# print("-gig", self.attributes[attributeId])
|
||||
for vc, vids in zip(attributeValsCount, attributeValsIds):
|
||||
# print("-gig", vids)
|
||||
gain -= vc/len(sampleIds) * self.getEntropy(vids)
|
||||
return gain
|
||||
|
||||
def getAttributeMaxInformationGain(self, sampleIds, attributeIds):
|
||||
attributesEntropy = [0] * len(attributeIds)
|
||||
for i, attId in zip(range(len(attributeIds)), attributeIds):
|
||||
attributesEntropy[i] = self.getInformationGain(sampleIds, attId)
|
||||
maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))]
|
||||
return self.attributes[maxId], maxId
|
||||
|
||||
def isSingleLabeled(self, sampleIds):
|
||||
label = self.labels[sampleIds[0]]
|
||||
for sid in sampleIds:
|
||||
if self.labels[sid] != label:
|
||||
return False
|
||||
return True
|
||||
|
||||
def getLabel(self, sampleId):
|
||||
return self.labels[sampleId]
|
||||
|
||||
def id3(self):
|
||||
sampleIds = [x for x in range(len(self.sample))]
|
||||
attributeIds = [x for x in range(len(self.attributes))]
|
||||
self.root = self.id3Recv(sampleIds, attributeIds, self.root)
|
||||
|
||||
def id3Recv(self, sampleIds, attributeIds, root):
|
||||
root = Node() # Initialize current root
|
||||
if self.isSingleLabeled(sampleIds):
|
||||
root.value = self.labels[sampleIds[0]]
|
||||
return root
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) == 0:
|
||||
root.value = self.getDominantLabel(sampleIds)
|
||||
return root
|
||||
bestAttrName, bestAttrId = self.getAttributeMaxInformationGain(
|
||||
sampleIds, attributeIds)
|
||||
# print(bestAttrName)
|
||||
root.value = bestAttrName
|
||||
root.childs = [] # Create list of children
|
||||
for value in self.getAttributeValues(sampleIds, bestAttrId):
|
||||
# print(value)
|
||||
child = Node()
|
||||
child.value = value
|
||||
root.childs.append(child) # Append new child node to current
|
||||
# root
|
||||
childSampleIds = []
|
||||
for sid in sampleIds:
|
||||
if self.sample[sid][bestAttrId] == value:
|
||||
childSampleIds.append(sid)
|
||||
if len(childSampleIds) == 0:
|
||||
child.next = self.getDominantLabel(sampleIds)
|
||||
else:
|
||||
# print(bestAttrName, bestAttrId)
|
||||
# print(attributeIds)
|
||||
if len(attributeIds) > 0 and bestAttrId in attributeIds:
|
||||
toRemove = attributeIds.index(bestAttrId)
|
||||
attributeIds.pop(toRemove)
|
||||
child.next = self.id3Recv(
|
||||
childSampleIds, attributeIds, child.next)
|
||||
return root
|
||||
|
||||
def printTree(self):
|
||||
if self.root:
|
||||
roots = deque()
|
||||
roots.append(self.root)
|
||||
while len(roots) > 0:
|
||||
root = roots.popleft()
|
||||
print(root.value)
|
||||
if root.childs:
|
||||
for child in root.childs:
|
||||
print('({})'.format(child.value))
|
||||
roots.append(child.next)
|
||||
elif root.next:
|
||||
print(root.next)
|
||||
|
||||
|
||||
def test():
|
||||
f = open('DataFiles/rideclass.csv')
|
||||
attributes = f.readline().split(',')
|
||||
attributes = attributes[1:len(attributes)-1]
|
||||
print(attributes)
|
||||
sample = f.readlines()
|
||||
f.close()
|
||||
for i in range(len(sample)):
|
||||
sample[i] = re.sub('\d+,', '', sample[i])
|
||||
sample[i] = sample[i].strip().split(',')
|
||||
labels = []
|
||||
for s in sample:
|
||||
labels.append(s.pop())
|
||||
# print(sample)
|
||||
# print(labels)
|
||||
decisionTree = DecisionTree(sample, attributes, labels)
|
||||
print("System entropy {}".format(decisionTree.entropy))
|
||||
decisionTree.id3()
|
||||
decisionTree.printTree()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
import numpy as np
|
||||
class DecisionTreeNode:
|
||||
def __init__(self, feature_index=None, threshold=None, left=None, right=None, output=None):
|
||||
self.feature_index = feature_index
|
||||
self.threshold = threshold
|
||||
self.left = left
|
||||
self.right = right
|
||||
self.output = output
|
||||
class DecisionTree:
|
||||
def __init__(self, min_samples_split=2, max_depth=5):
|
||||
self.root = None
|
||||
self.min_samples_split = min_samples_split
|
||||
self.max_depth = max_depth
|
||||
def fit(self, X, y):
|
||||
self.root = self._grow_tree(X, y)
|
||||
def _grow_tree(self, X, y, depth=0):
|
||||
num_samples, num_features = X.shape
|
||||
unique_classes = np.unique(y)
|
||||
# Check for stopping conditions
|
||||
if (num_samples < self.min_samples_split) or (depth == self.max_depth) or (len(unique_classes) == 1):
|
||||
output = self._most_common_label(y)
|
||||
return DecisionTreeNode(output=output)
|
||||
# Find the best split
|
||||
best_feature, best_threshold = self._best_split(X, y, num_features)
|
||||
left_indices = X[:, best_feature] < best_threshold
|
||||
right_indices = X[:, best_feature] >= best_threshold
|
||||
left_child = self._grow_tree(X[left_indices], y[left_indices], depth + 1)
|
||||
right_child = self._grow_tree(X[right_indices], y[right_indices], depth + 1)
|
||||
return DecisionTreeNode(feature_index=best_feature, threshold=best_threshold, left=left_child, right=right_child)
|
||||
def _best_split(self, X, y, num_features):
|
||||
best_gain = -1
|
||||
best_feature, best_threshold = None, None
|
||||
for feature in range(num_features):
|
||||
thresholds, classes = zip(*sorted(zip(X[:, feature], y)))
|
||||
num_left = [0] * len(np.unique(y))
|
||||
num_right = [np.sum(classes == c) for c in np.unique(y)]
|
||||
for i in range(1, len(y)): # At least one in each side
|
||||
c = classes[i - 1]
|
||||
num_left[c] += 1
|
||||
num_right[c] -= 1
|
||||
gain = self._information_gain(num_left, num_right, len(classes), len(y))
|
||||
if thresholds[i] == thresholds[i - 1]: # Skip duplicate values
|
||||
continue
|
||||
if gain > best_gain:
|
||||
best_gain = gain
|
||||
best_feature = feature
|
||||
best_threshold = (thresholds[i] + thresholds[i - 1]) / 2 # Average threshold
|
||||
return best_feature, best_threshold
|
||||
def _information_gain(self, num_left, num_right, num_total, num_classes):
|
||||
p_left = float(len(num_left)) / num_total
|
||||
p_right = float(len(num_right)) / num_total
|
||||
entropy_before = self._entropy(num_left, num_total)
|
||||
entropy_left = self._entropy(num_left, sum(num_left))
|
||||
entropy_right = self._entropy(num_right, sum(num_right))
|
||||
entropy_after = p_left * entropy_left + p_right * entropy_right
|
||||
|
||||
return entropy_before - entropy_after
|
||||
def _entropy(self, counts, total):
|
||||
if total == 0:
|
||||
return 0
|
||||
return -sum((count / total) * np.log2(count / total) for count in counts if count > 0)
|
||||
def _most_common_label(self, y):
|
||||
return np.bincount(y).argmax()
|
||||
def predict(self, X):
|
||||
return np.array([self._predict_sample(sample, self.root) for sample in X])
|
||||
def _predict_sample(self, sample, node):
|
||||
if node.output is not None:
|
||||
return node.output
|
||||
if sample[node.feature_index] < node.threshold:
|
||||
return self._predict_sample(sample, node.left)
|
||||
else:
|
||||
return self._predict_sample(sample, node.right)
|
||||
# Example usage
|
||||
if __name__ == "__main__":
|
||||
# Sample data (AND gate)
|
||||
X = np.array([[0, 0], [1, 0], [0, 1], [1, 1]])
|
||||
y = np.array([0, 0, 0, 1])
|
||||
model = DecisionTree(min_samples_split=1, max_depth=3)
|
||||
model.fit(X, y)
|
||||
predictions = model.predict(X)
|
||||
print("Predictions:", predictions)
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import numpy as np
|
||||
class DecisionTreeRegressor:
|
||||
def __init__(self, max_depth=3):
|
||||
self.max_depth = max_depth
|
||||
self.tree = None
|
||||
def fit(self, X, y):
|
||||
self.tree = self._grow_tree(X, y)
|
||||
def _grow_tree(self, X, y, depth=0):
|
||||
n_samples, n_features = X.shape
|
||||
if depth < self.max_depth:
|
||||
best_feature, best_threshold = self._best_split(X, y)
|
||||
if best_feature is not None:
|
||||
left_indices = X[:, best_feature] < best_threshold
|
||||
right_indices = X[:, best_feature] >= best_threshold
|
||||
left_child = self._grow_tree(X[left_indices], y[left_indices], depth + 1)
|
||||
right_child = self._grow_tree(X[right_indices], y[right_indices], depth + 1)
|
||||
return (best_feature, best_threshold, left_child, right_child)
|
||||
return np.mean(y)
|
||||
def _best_split(self, X, y):
|
||||
best_mse = float('inf')
|
||||
best_feature, best_threshold = None, None
|
||||
n_samples, n_features = X.shape
|
||||
|
||||
for feature in range(n_features):
|
||||
thresholds = np.unique(X[:, feature])
|
||||
for threshold in thresholds:
|
||||
left_indices = X[:, feature] < threshold
|
||||
right_indices = X[:, feature] >= threshold
|
||||
if len(y[left_indices]) > 0 and len(y[right_indices]) > 0:
|
||||
left_mse = np.mean((y[left_indices] - np.mean(y[left_indices])) ** 2)
|
||||
right_mse = np.mean((y[right_indices] - np.mean(y[right_indices])) ** 2)
|
||||
mse = (len(y[left_indices]) * left_mse + len(y[right_indices]) * right_mse) / n_samples
|
||||
|
||||
if mse < best_mse:
|
||||
best_mse = mse
|
||||
best_feature = feature
|
||||
best_threshold = threshold
|
||||
return best_feature, best_threshold
|
||||
def predict(self, X):
|
||||
return np.array([self._predict_sample(sample, self.tree) for sample in X])
|
||||
def _predict_sample(self, sample, node):
|
||||
if isinstance(node, tuple):
|
||||
feature, threshold, left_child, right_child = node
|
||||
if sample[feature] < threshold:
|
||||
return self._predict_sample(sample, left_child)
|
||||
else:
|
||||
return self._predict_sample(sample, right_child)
|
||||
return node
|
||||
class GradientBoostingRegressor:
|
||||
def __init__(self, n_estimators=100, learning_rate=0.1, max_depth=3):
|
||||
self.n_estimators = n_estimators
|
||||
self.learning_rate = learning_rate
|
||||
self.max_depth = max_depth
|
||||
self.models = []
|
||||
def fit(self, X, y):
|
||||
y_pred = np.zeros(y.shape)
|
||||
for _ in range(self.n_estimators):
|
||||
residuals = y - y_pred
|
||||
model = DecisionTreeRegressor(max_depth=self.max_depth)
|
||||
model.fit(X, residuals)
|
||||
y_pred += self.learning_rate * model.predict(X)
|
||||
self.models.append(model)
|
||||
def predict(self, X):
|
||||
y_pred = np.zeros(X.shape[0])
|
||||
for model in self.models:
|
||||
y_pred += self.learning_rate * model.predict(X)
|
||||
return y_pred
|
||||
# Example usage
|
||||
if __name__ == "__main__":
|
||||
# Sample data
|
||||
X = np.array([[1], [2], [3], [4], [5]])
|
||||
y = np.array([1.5, 1.7, 3.5, 3.7, 5.0])
|
||||
model = GradientBoostingRegressor(n_estimators=100, learning_rate=0.1, max_depth=2)
|
||||
model.fit(X, y)
|
||||
predictions = model.predict(X)
|
||||
print("Predictions:", predictions)
|
||||
Reference in New Issue
Block a user