From fae683c3f12da2311929fbd3d96ebfb015a9bd80 Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Sun, 29 Sep 2024 21:58:56 +0200 Subject: [PATCH] added codes --- doc/src/DecisionTrees/Programs/adagrad.py | 27 ++ .../DecisionTrees/Programs/decisiontree.py | 268 ++++++------------ .../Programs/gradientboosting.py | 76 +++++ doc/src/week40/codes/lasso.py | 26 ++ doc/src/week40/codes/logreg.py | 32 +++ doc/src/week40/codes/perceptron.py | 37 +++ doc/src/week40/codes/regression.py | 107 +++++++ doc/src/week40/codes/test.py | 19 ++ 8 files changed, 405 insertions(+), 187 deletions(-) create mode 100644 doc/src/DecisionTrees/Programs/adagrad.py create mode 100644 doc/src/DecisionTrees/Programs/gradientboosting.py create mode 100644 doc/src/week40/codes/lasso.py create mode 100644 doc/src/week40/codes/logreg.py create mode 100644 doc/src/week40/codes/perceptron.py create mode 100644 doc/src/week40/codes/regression.py create mode 100644 doc/src/week40/codes/test.py diff --git a/doc/src/DecisionTrees/Programs/adagrad.py b/doc/src/DecisionTrees/Programs/adagrad.py new file mode 100644 index 000000000..b82d3df80 --- /dev/null +++ b/doc/src/DecisionTrees/Programs/adagrad.py @@ -0,0 +1,27 @@ +import numpy as np +class AdaGrad: + def __init__(self, learning_rate=0.01, epsilon=1e-8): + self.learning_rate = learning_rate + self.epsilon = epsilon + self.gradient_squared = None + def update(self, weights, gradient): + if self.gradient_squared is None: + self.gradient_squared = np.zeros_like(weights) + # Accumulate squared gradients + self.gradient_squared += gradient ** 2 + + # Update weights + adjusted_grads = gradient / (np.sqrt(self.gradient_squared) + self.epsilon) + weights -= self.learning_rate * adjusted_grads + + return weights +# Example usage +if __name__ == "__main__": + # Sample data and gradient + np.random.seed(0) + weights = np.random.rand(3) + gradients = np.random.rand(100, 3) # Simulating 100 gradients + optimizer = AdaGrad(learning_rate=0.1) + for grad in gradients: + weights = optimizer.update(weights, grad) + print("Updated weights:", weights) diff --git a/doc/src/DecisionTrees/Programs/decisiontree.py b/doc/src/DecisionTrees/Programs/decisiontree.py index 1142d2d84..1bf589184 100644 --- a/doc/src/DecisionTrees/Programs/decisiontree.py +++ b/doc/src/DecisionTrees/Programs/decisiontree.py @@ -1,187 +1,81 @@ -import re -import math -from collections import deque - -# x is examples in training set -# y is set of attributes -# label is target attributes -# Node is a class which has properties values, childs, and next -# root is top node in the decision tree - -class Node(object): - def __init__(self): - self.value = None - self.next = None - self.childs = None - -# Simple class of Decision Tree -# Aimed for who want to learn Decision Tree, so it is not optimized -class DecisionTree(object): - def __init__(self, sample, attributes, labels): - self.sample = sample - self.attributes = attributes - self.labels = labels - self.labelCodes = None - self.labelCodesCount = None - self.initLabelCodes() - # print(self.labelCodes) - self.root = None - self.entropy = self.getEntropy([x for x in range(len(self.labels))]) - - def initLabelCodes(self): - self.labelCodes = [] - self.labelCodesCount = [] - for l in self.labels: - if l not in self.labelCodes: - self.labelCodes.append(l) - self.labelCodesCount.append(0) - self.labelCodesCount[self.labelCodes.index(l)] += 1 - - def getLabelCodeId(self, sampleId): - return self.labelCodes.index(self.labels[sampleId]) - - def getAttributeValues(self, sampleIds, attributeId): - vals = [] - for sid in sampleIds: - val = self.sample[sid][attributeId] - if val not in vals: - vals.append(val) - # print(vals) - return vals - - def getEntropy(self, sampleIds): - entropy = 0 - labelCount = [0] * len(self.labelCodes) - for sid in sampleIds: - labelCount[self.getLabelCodeId(sid)] += 1 - # print("-ge", labelCount) - for lv in labelCount: - # print(lv) - if lv != 0: - entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2) - else: - entropy += 0 - return entropy - - def getDominantLabel(self, sampleIds): - labelCodesCount = [0] * len(self.labelCodes) - for sid in sampleIds: - labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1 - return self.labelCodes[labelCodesCount.index(max(labelCodesCount))] - - def getInformationGain(self, sampleIds, attributeId): - gain = self.getEntropy(sampleIds) - attributeVals = [] - attributeValsCount = [] - attributeValsIds = [] - for sid in sampleIds: - val = self.sample[sid][attributeId] - if val not in attributeVals: - attributeVals.append(val) - attributeValsCount.append(0) - attributeValsIds.append([]) - vid = attributeVals.index(val) - attributeValsCount[vid] += 1 - attributeValsIds[vid].append(sid) - # print("-gig", self.attributes[attributeId]) - for vc, vids in zip(attributeValsCount, attributeValsIds): - # print("-gig", vids) - gain -= vc/len(sampleIds) * self.getEntropy(vids) - return gain - - def getAttributeMaxInformationGain(self, sampleIds, attributeIds): - attributesEntropy = [0] * len(attributeIds) - for i, attId in zip(range(len(attributeIds)), attributeIds): - attributesEntropy[i] = self.getInformationGain(sampleIds, attId) - maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))] - return self.attributes[maxId], maxId - - def isSingleLabeled(self, sampleIds): - label = self.labels[sampleIds[0]] - for sid in sampleIds: - if self.labels[sid] != label: - return False - return True - - def getLabel(self, sampleId): - return self.labels[sampleId] - - def id3(self): - sampleIds = [x for x in range(len(self.sample))] - attributeIds = [x for x in range(len(self.attributes))] - self.root = self.id3Recv(sampleIds, attributeIds, self.root) - - def id3Recv(self, sampleIds, attributeIds, root): - root = Node() # Initialize current root - if self.isSingleLabeled(sampleIds): - root.value = self.labels[sampleIds[0]] - return root - # print(attributeIds) - if len(attributeIds) == 0: - root.value = self.getDominantLabel(sampleIds) - return root - bestAttrName, bestAttrId = self.getAttributeMaxInformationGain( - sampleIds, attributeIds) - # print(bestAttrName) - root.value = bestAttrName - root.childs = [] # Create list of children - for value in self.getAttributeValues(sampleIds, bestAttrId): - # print(value) - child = Node() - child.value = value - root.childs.append(child) # Append new child node to current - # root - childSampleIds = [] - for sid in sampleIds: - if self.sample[sid][bestAttrId] == value: - childSampleIds.append(sid) - if len(childSampleIds) == 0: - child.next = self.getDominantLabel(sampleIds) - else: - # print(bestAttrName, bestAttrId) - # print(attributeIds) - if len(attributeIds) > 0 and bestAttrId in attributeIds: - toRemove = attributeIds.index(bestAttrId) - attributeIds.pop(toRemove) - child.next = self.id3Recv( - childSampleIds, attributeIds, child.next) - return root - - def printTree(self): - if self.root: - roots = deque() - roots.append(self.root) - while len(roots) > 0: - root = roots.popleft() - print(root.value) - if root.childs: - for child in root.childs: - print('({})'.format(child.value)) - roots.append(child.next) - elif root.next: - print(root.next) - - -def test(): - f = open('DataFiles/rideclass.csv') - attributes = f.readline().split(',') - attributes = attributes[1:len(attributes)-1] - print(attributes) - sample = f.readlines() - f.close() - for i in range(len(sample)): - sample[i] = re.sub('\d+,', '', sample[i]) - sample[i] = sample[i].strip().split(',') - labels = [] - for s in sample: - labels.append(s.pop()) - # print(sample) - # print(labels) - decisionTree = DecisionTree(sample, attributes, labels) - print("System entropy {}".format(decisionTree.entropy)) - decisionTree.id3() - decisionTree.printTree() - - -if __name__ == '__main__': - test() +import numpy as np +class DecisionTreeNode: + def __init__(self, feature_index=None, threshold=None, left=None, right=None, output=None): + self.feature_index = feature_index + self.threshold = threshold + self.left = left + self.right = right + self.output = output +class DecisionTree: + def __init__(self, min_samples_split=2, max_depth=5): + self.root = None + self.min_samples_split = min_samples_split + self.max_depth = max_depth + def fit(self, X, y): + self.root = self._grow_tree(X, y) + def _grow_tree(self, X, y, depth=0): + num_samples, num_features = X.shape + unique_classes = np.unique(y) + # Check for stopping conditions + if (num_samples < self.min_samples_split) or (depth == self.max_depth) or (len(unique_classes) == 1): + output = self._most_common_label(y) + return DecisionTreeNode(output=output) + # Find the best split + best_feature, best_threshold = self._best_split(X, y, num_features) + left_indices = X[:, best_feature] < best_threshold + right_indices = X[:, best_feature] >= best_threshold + left_child = self._grow_tree(X[left_indices], y[left_indices], depth + 1) + right_child = self._grow_tree(X[right_indices], y[right_indices], depth + 1) + return DecisionTreeNode(feature_index=best_feature, threshold=best_threshold, left=left_child, right=right_child) + def _best_split(self, X, y, num_features): + best_gain = -1 + best_feature, best_threshold = None, None + for feature in range(num_features): + thresholds, classes = zip(*sorted(zip(X[:, feature], y))) + num_left = [0] * len(np.unique(y)) + num_right = [np.sum(classes == c) for c in np.unique(y)] + for i in range(1, len(y)): # At least one in each side + c = classes[i - 1] + num_left[c] += 1 + num_right[c] -= 1 + gain = self._information_gain(num_left, num_right, len(classes), len(y)) + if thresholds[i] == thresholds[i - 1]: # Skip duplicate values + continue + if gain > best_gain: + best_gain = gain + best_feature = feature + best_threshold = (thresholds[i] + thresholds[i - 1]) / 2 # Average threshold + return best_feature, best_threshold + def _information_gain(self, num_left, num_right, num_total, num_classes): + p_left = float(len(num_left)) / num_total + p_right = float(len(num_right)) / num_total + entropy_before = self._entropy(num_left, num_total) + entropy_left = self._entropy(num_left, sum(num_left)) + entropy_right = self._entropy(num_right, sum(num_right)) + entropy_after = p_left * entropy_left + p_right * entropy_right + + return entropy_before - entropy_after + def _entropy(self, counts, total): + if total == 0: + return 0 + return -sum((count / total) * np.log2(count / total) for count in counts if count > 0) + def _most_common_label(self, y): + return np.bincount(y).argmax() + def predict(self, X): + return np.array([self._predict_sample(sample, self.root) for sample in X]) + def _predict_sample(self, sample, node): + if node.output is not None: + return node.output + if sample[node.feature_index] < node.threshold: + return self._predict_sample(sample, node.left) + else: + return self._predict_sample(sample, node.right) +# Example usage +if __name__ == "__main__": + # Sample data (AND gate) + X = np.array([[0, 0], [1, 0], [0, 1], [1, 1]]) + y = np.array([0, 0, 0, 1]) + model = DecisionTree(min_samples_split=1, max_depth=3) + model.fit(X, y) + predictions = model.predict(X) + print("Predictions:", predictions) diff --git a/doc/src/DecisionTrees/Programs/gradientboosting.py b/doc/src/DecisionTrees/Programs/gradientboosting.py new file mode 100644 index 000000000..bf4ba8d7c --- /dev/null +++ b/doc/src/DecisionTrees/Programs/gradientboosting.py @@ -0,0 +1,76 @@ +import numpy as np +class DecisionTreeRegressor: + def __init__(self, max_depth=3): + self.max_depth = max_depth + self.tree = None + def fit(self, X, y): + self.tree = self._grow_tree(X, y) + def _grow_tree(self, X, y, depth=0): + n_samples, n_features = X.shape + if depth < self.max_depth: + best_feature, best_threshold = self._best_split(X, y) + if best_feature is not None: + left_indices = X[:, best_feature] < best_threshold + right_indices = X[:, best_feature] >= best_threshold + left_child = self._grow_tree(X[left_indices], y[left_indices], depth + 1) + right_child = self._grow_tree(X[right_indices], y[right_indices], depth + 1) + return (best_feature, best_threshold, left_child, right_child) + return np.mean(y) + def _best_split(self, X, y): + best_mse = float('inf') + best_feature, best_threshold = None, None + n_samples, n_features = X.shape + + for feature in range(n_features): + thresholds = np.unique(X[:, feature]) + for threshold in thresholds: + left_indices = X[:, feature] < threshold + right_indices = X[:, feature] >= threshold + if len(y[left_indices]) > 0 and len(y[right_indices]) > 0: + left_mse = np.mean((y[left_indices] - np.mean(y[left_indices])) ** 2) + right_mse = np.mean((y[right_indices] - np.mean(y[right_indices])) ** 2) + mse = (len(y[left_indices]) * left_mse + len(y[right_indices]) * right_mse) / n_samples + + if mse < best_mse: + best_mse = mse + best_feature = feature + best_threshold = threshold + return best_feature, best_threshold + def predict(self, X): + return np.array([self._predict_sample(sample, self.tree) for sample in X]) + def _predict_sample(self, sample, node): + if isinstance(node, tuple): + feature, threshold, left_child, right_child = node + if sample[feature] < threshold: + return self._predict_sample(sample, left_child) + else: + return self._predict_sample(sample, right_child) + return node +class GradientBoostingRegressor: + def __init__(self, n_estimators=100, learning_rate=0.1, max_depth=3): + self.n_estimators = n_estimators + self.learning_rate = learning_rate + self.max_depth = max_depth + self.models = [] + def fit(self, X, y): + y_pred = np.zeros(y.shape) + for _ in range(self.n_estimators): + residuals = y - y_pred + model = DecisionTreeRegressor(max_depth=self.max_depth) + model.fit(X, residuals) + y_pred += self.learning_rate * model.predict(X) + self.models.append(model) + def predict(self, X): + y_pred = np.zeros(X.shape[0]) + for model in self.models: + y_pred += self.learning_rate * model.predict(X) + return y_pred +# Example usage +if __name__ == "__main__": + # Sample data + X = np.array([[1], [2], [3], [4], [5]]) + y = np.array([1.5, 1.7, 3.5, 3.7, 5.0]) + model = GradientBoostingRegressor(n_estimators=100, learning_rate=0.1, max_depth=2) + model.fit(X, y) + predictions = model.predict(X) + print("Predictions:", predictions) diff --git a/doc/src/week40/codes/lasso.py b/doc/src/week40/codes/lasso.py new file mode 100644 index 000000000..088be3ecc --- /dev/null +++ b/doc/src/week40/codes/lasso.py @@ -0,0 +1,26 @@ +import numpy as np +class LassoRegression: + def __init__(self, learning_rate=0.01, num_iterations=1000, lambda_reg=1.0): + self.learning_rate = learning_rate + self.num_iterations = num_iterations + self.lambda_reg = lambda_reg + self.weights = None + def fit(self, X, y): + num_samples, num_features = X.shape + self.weights = np.zeros(num_features) + for _ in range(self.num_iterations): + linear_model = np.dot(X, self.weights) + gradient = (1 / num_samples) * np.dot(X.T, (linear_model - y)) + self.lambda_reg * np.sign(self.weights) + # Update weights + self.weights -= self.learning_rate * gradient + def predict(self, X): + return np.dot(X, self.weights) +# Example usage +if __name__ == "__main__": + # Sample data + X = np.array([[1, 2], [2, 3], [3, 4], [4, 5]]) + y = np.array([1, 2, 3, 4]) + model = LassoRegression(learning_rate=0.01, num_iterations=1000, lambda_reg=0.1) + model.fit(X, y) + predictions = model.predict(X) + print("Predictions:", predictions) diff --git a/doc/src/week40/codes/logreg.py b/doc/src/week40/codes/logreg.py new file mode 100644 index 000000000..687a0d7f6 --- /dev/null +++ b/doc/src/week40/codes/logreg.py @@ -0,0 +1,32 @@ +import numpy as np +class LogisticRegression: + def __init__(self, learning_rate=0.01, num_iterations=1000): + self.learning_rate = learning_rate + self.num_iterations = num_iterations + self.weights = None + def sigmoid(self, z): + return 1 / (1 + np.exp(-z)) + def fit(self, X, y): + num_samples, num_features = X.shape + self.weights = np.zeros(num_features) + for _ in range(self.num_iterations): + linear_model = np.dot(X, self.weights) + y_predicted = self.sigmoid(linear_model) + # Gradient calculation + gradient = np.dot(X.T, (y_predicted - y)) / num_samples + + # Update weights + self.weights -= self.learning_rate * gradient + def predict(self, X): + linear_model = np.dot(X, self.weights) + y_predicted = self.sigmoid(linear_model) + return [1 if i >= 0.5 else 0 for i in y_predicted] +# Example usage +if __name__ == "__main__": + # Sample data + X = np.array([[0, 0], [1, 0], [0, 1], [1, 1]]) + y = np.array([0, 0, 0, 1]) # AND gate + model = LogisticRegression(learning_rate=0.1, num_iterations=1000) + model.fit(X, y) + predictions = model.predict(X) + print("Predictions:", predictions) diff --git a/doc/src/week40/codes/perceptron.py b/doc/src/week40/codes/perceptron.py new file mode 100644 index 000000000..4c501c500 --- /dev/null +++ b/doc/src/week40/codes/perceptron.py @@ -0,0 +1,37 @@ +import numpy as np +class Perceptron: + def __init__(self, learning_rate=0.01, n_iters=1000): + self.learning_rate = learning_rate + self.n_iters = n_iters + self.weights = None + self.bias = None + def fit(self, X, y): + n_samples, n_features = X.shape + self.weights = np.zeros(n_features) + self.bias = 0 + for _ in range(self.n_iters): + for idx, x_i in enumerate(X): + linear_output = np.dot(x_i, self.weights) + self.bias + y_predicted = self.activation_function(linear_output) + # Update weights and bias + update = self.learning_rate * (y[idx] - y_predicted) + self.weights += update * x_i + self.bias += update + def activation_function(self, x): + return 1 if x >= 0 else 0 + def predict(self, X): + linear_output = np.dot(X, self.weights) + self.bias + y_predicted = [self.activation_function(i) for i in linear_output] + return np.array(y_predicted) +# Example usage +if __name__ == "__main__": + # Sample data (AND logic gate) + X = np.array([[0, 0], + [0, 1], + [1, 0], + [1, 1]]) + y = np.array([0, 0, 0, 1]) # AND outputs + perceptron = Perceptron(learning_rate=0.1, n_iters=10) + perceptron.fit(X, y) + predictions = perceptron.predict(X) + print("Final predictions:", predictions) diff --git a/doc/src/week40/codes/regression.py b/doc/src/week40/codes/regression.py new file mode 100644 index 000000000..c18bcaf50 --- /dev/null +++ b/doc/src/week40/codes/regression.py @@ -0,0 +1,107 @@ +# Importing various packages +from random import random, seed +import numpy as np +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d import Axes3D +from matplotlib import cm +from matplotlib.ticker import LinearLocator, FormatStrFormatter +import sys + +# the number of datapoints +n = 100 +x = 2*np.random.rand(n,1) +y = 4+3*x+np.random.randn(n,1) + +X = np.c_[np.ones((n,1)), x] +# Hessian matrix +H = (2.0/n)* X.T @ X +# Get the eigenvalues +EigValues, EigVectors = np.linalg.eig(H) +print(f"Eigenvalues of Hessian Matrix:{EigValues}") + +beta_linreg = np.linalg.pinv(X.T @ X) @ X.T @ y +print(beta_linreg) +beta = np.random.randn(2,1) + +eta = 1.0/np.max(EigValues) +Niterations = 1000 + +for iter in range(Niterations): + gradient = (2.0/n)*X.T @ (X @ beta-y) + beta -= eta*gradient + +print(beta) +xnew = np.array([[0],[2]]) +xbnew = np.c_[np.ones((2,1)), xnew] +ypredict = xbnew.dot(beta) +ypredict2 = xbnew.dot(beta_linreg) +plt.plot(xnew, ypredict, "r-") +plt.plot(xnew, ypredict2, "b-") +plt.plot(x, y ,'ro') +plt.axis([0,2.0,0, 15.0]) +plt.xlabel(r'$x$') +plt.ylabel(r'$y$') +plt.title(r'Gradient descent example') +plt.show() + + + +X = np.c_[np.ones((n,1)), x] +XT_X = X.T @ X + +#Ridge parameter lambda +lmbda = 0.001 +Id = n*lmbda* np.eye(XT_X.shape[0]) + +# Hessian matrix +H = (2.0/n)* XT_X+2*lmbda* np.eye(XT_X.shape[0]) +# Get the eigenvalues +EigValues, EigVectors = np.linalg.eig(H) +print(f"Eigenvalues of Hessian Matrix:{EigValues}") + + +beta_linreg = np.linalg.pinv(XT_X+Id) @ X.T @ y +print(beta_linreg) +# Start plain gradient descent +beta = np.random.randn(2,1) + +eta = 1.0/np.max(EigValues) +Niterations = 100 + +for iter in range(Niterations): + gradients = 2.0/n*X.T @ (X @ (beta)-y)+2*lmbda*beta + beta -= eta*gradients + +print(beta) +ypredict = X @ beta +ypredict2 = X @ beta_linreg +plt.plot(x, ypredict, "r-") +plt.plot(x, ypredict2, "b-") +plt.plot(x, y ,'ro') +plt.axis([0,2.0,0, 15.0]) +plt.xlabel(r'$x$') +plt.ylabel(r'$y$') +plt.title(r'Gradient descent example for Ridge') +plt.show() + + +# And now with Lasso +# Start plain gradient descent +beta_lasso = np.random.randn(2,1) + +eta = 0.01 +Niterations = 100 +for iter in range(Niterations): + gradients = 2.0/n*X.T @ (X @ (beta)-y)+2*lmbda*np.sign(beta) + beta_lasso -= eta*gradients + +print('Gradient descent with Lasso:', beta_lasso) +ypredict = X @ beta_lasso +plt.plot(x, ypredict, "r-") +plt.plot(x, y ,'ro') +plt.axis([0,2.0,0, 15.0]) +plt.xlabel(r'$x$') +plt.ylabel(r'$y$') +plt.title(r'Gradient descent example for Lasso') +plt.show() + diff --git a/doc/src/week40/codes/test.py b/doc/src/week40/codes/test.py new file mode 100644 index 000000000..3dbd8673f --- /dev/null +++ b/doc/src/week40/codes/test.py @@ -0,0 +1,19 @@ +import Quanthon as qt +#Initializing a Single Qubit + +#Initialize a single qubit by creating an instance of the Qubits class. + +qubit = qt.Qubits(1) +# Apply a Hadamard gate on the first qubit +qubit.H(0) + +# Apply a Pauli-X gate on the first qubit +qubit.X(0) + +# Apply a Pauli-Y gate on the first qubit +qubit.Y(0) + +# Apply a Pauli-Z gate on the first qubit +qubit.Z(0) + +result = qubit.measure(n_shots=10)