From a26b5df8a5ecb7129538c579efbe9823f69f6065 Mon Sep 17 00:00:00 2001 From: mhjensen Date: Sun, 17 Nov 2019 17:10:50 +0100 Subject: [PATCH] added codes --- .../SupportVMachines/Programs/adaboost.py~ | 65 ++++++ .../SupportVMachines/Programs/bootstrap.py | 36 ++++ .../SupportVMachines/Programs/bootstrap.py~ | 56 ++++++ doc/src/SupportVMachines/Programs/cancer.py | 41 ++++ doc/src/SupportVMachines/Programs/cart.py | 172 ++++++++++++++++ .../SupportVMachines/Programs/decisiontree.py | 187 ++++++++++++++++++ doc/src/SupportVMachines/Programs/dtcancer.py | 57 ++++++ doc/src/SupportVMachines/Programs/gdclas.py | 37 ++++ doc/src/SupportVMachines/Programs/gdreg.py | 48 +++++ doc/src/SupportVMachines/Programs/gdreg.py~ | 48 +++++ doc/src/SupportVMachines/Programs/gini.py | 74 +++++++ doc/src/SupportVMachines/Programs/moons.py | 28 +++ .../Programs/rfbootclassify.py | 54 +++++ .../Programs/rfbootclassify.py~ | 54 +++++ .../SupportVMachines/Programs/rfbootstrap.py | 55 ++++++ .../SupportVMachines/Programs/rfbootstrap.py~ | 54 +++++ doc/src/SupportVMachines/Programs/rfcancer.py | 39 ++++ doc/src/SupportVMachines/Programs/ride.py | 68 +++++++ doc/src/SupportVMachines/Programs/xgcancer.py | 46 +++++ .../SupportVMachines/Programs/xgcancer.py~ | 41 ++++ .../SupportVMachines/Programs/xgregressor.py | 65 ++++++ .../SupportVMachines/Programs/xgregressor.py~ | 47 +++++ 22 files changed, 1372 insertions(+) create mode 100644 doc/src/SupportVMachines/Programs/adaboost.py~ create mode 100644 doc/src/SupportVMachines/Programs/bootstrap.py create mode 100644 doc/src/SupportVMachines/Programs/bootstrap.py~ create mode 100644 doc/src/SupportVMachines/Programs/cancer.py create mode 100644 doc/src/SupportVMachines/Programs/cart.py create mode 100644 doc/src/SupportVMachines/Programs/decisiontree.py create mode 100644 doc/src/SupportVMachines/Programs/dtcancer.py create mode 100644 doc/src/SupportVMachines/Programs/gdclas.py create mode 100644 doc/src/SupportVMachines/Programs/gdreg.py create mode 100644 doc/src/SupportVMachines/Programs/gdreg.py~ create mode 100644 doc/src/SupportVMachines/Programs/gini.py create mode 100644 doc/src/SupportVMachines/Programs/moons.py create mode 100644 doc/src/SupportVMachines/Programs/rfbootclassify.py create mode 100644 doc/src/SupportVMachines/Programs/rfbootclassify.py~ create mode 100644 doc/src/SupportVMachines/Programs/rfbootstrap.py create mode 100644 doc/src/SupportVMachines/Programs/rfbootstrap.py~ create mode 100644 doc/src/SupportVMachines/Programs/rfcancer.py create mode 100644 doc/src/SupportVMachines/Programs/ride.py create mode 100644 doc/src/SupportVMachines/Programs/xgcancer.py create mode 100644 doc/src/SupportVMachines/Programs/xgcancer.py~ create mode 100644 doc/src/SupportVMachines/Programs/xgregressor.py create mode 100644 doc/src/SupportVMachines/Programs/xgregressor.py~ diff --git a/doc/src/SupportVMachines/Programs/adaboost.py~ b/doc/src/SupportVMachines/Programs/adaboost.py~ new file mode 100644 index 000000000..09eaec98c --- /dev/null +++ b/doc/src/SupportVMachines/Programs/adaboost.py~ @@ -0,0 +1,65 @@ +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.datasets import load_breast_cancer +from sklearn.svm import SVC +from sklearn.linear_model import LogisticRegression +from sklearn.tree import DecisionTreeClassifier +from sklearn.ensemble import AdaBoostClassifier + +def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True): + x1s = np.linspace(axes[0], axes[1], 100) + x2s = np.linspace(axes[2], axes[3], 100) + x1, x2 = np.meshgrid(x1s, x2s) + X_new = np.c_[x1.ravel(), x2.ravel()] + y_pred = clf.predict(X_new).reshape(x1.shape) + custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0']) + plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap) + if contour: + custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50']) + plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8) + plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha) + plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha) + plt.axis(axes) + plt.xlabel(r"$x_1$", fontsize=18) + plt.ylabel(r"$x_2$", fontsize=18, rotation=0) + +# Load the data +cancer = load_breast_cancer() +X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0) +#now scale the data +from sklearn.preprocessing import StandardScaler +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) + +ada_clf = AdaBoostClassifier( + DecisionTreeClassifier(max_depth=1), n_estimators=200, + algorithm="SAMME.R", learning_rate=0.5, random_state=42) + +ada_clf.fit(X_train_scaled, y_train) +plot_decision_boundary(ada_clf, cancer.data,cancer.target) + +m = len(X_train_scaled) + +plt.figure(figsize=(11, 4)) +for subplot, learning_rate in ((121, 1), (122, 0.5)): + sample_weights = np.ones(m) + plt.subplot(subplot) + for i in range(5): + svm_clf = SVC(kernel="rbf", C=0.05, gamma="auto", random_state=42) + svm_clf.fit(X_train_scaled, y_train, sample_weight=sample_weights) + y_pred = svm_clf.predict(X_train_scaled) + sample_weights[y_pred != y_train] *= (1 + learning_rate) + plot_decision_boundary(svm_clf, cancer.data,cancer.target, alpha=0.2) + plt.title("learning_rate = {}".format(learning_rate), fontsize=16) + if subplot == 121: + plt.text(-0.7, -0.65, "1", fontsize=14) + plt.text(-0.6, -0.10, "2", fontsize=14) + plt.text(-0.5, 0.10, "3", fontsize=14) + plt.text(-0.4, 0.55, "4", fontsize=14) + plt.text(-0.3, 0.90, "5", fontsize=14) + +plt.show() + diff --git a/doc/src/SupportVMachines/Programs/bootstrap.py b/doc/src/SupportVMachines/Programs/bootstrap.py new file mode 100644 index 000000000..017bb9ae3 --- /dev/null +++ b/doc/src/SupportVMachines/Programs/bootstrap.py @@ -0,0 +1,36 @@ +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.utils import resample +from sklearn.tree import DecisionTreeRegressor +from sklearn.datasets import load_breast_cancer + +# Load the data +cancer = load_breast_cancer() +maxdepth = 6 +X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0) +#now scale the data +from sklearn.preprocessing import StandardScaler +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) +score = np.zeros(maxdepth) +depth = np.zeros(maxdepth) + +for degree in range(1,maxdepth): + model = DecisionTreeRegressor(max_depth=degree) + model.fit(X_train_scaled,y_train) + y_pred = model.predict(X_test_scaled) + depth[degree] = degree + score[degree] = model.score(X_test_scaled,y_pred) + print('Max Tree Depth:', degree) + print('Score:', score[degree]) + +plt.xlim(1,maxdepth) +plt.plot(depth, score, label='Score') +plt.legend() +plt.show() + + + diff --git a/doc/src/SupportVMachines/Programs/bootstrap.py~ b/doc/src/SupportVMachines/Programs/bootstrap.py~ new file mode 100644 index 000000000..623de3af2 --- /dev/null +++ b/doc/src/SupportVMachines/Programs/bootstrap.py~ @@ -0,0 +1,56 @@ + +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.utils import resample +from sklearn.tree import DecisionTreeRegressor + + +np.random.seed(2018) + +n = 40 +n_boostraps = 100 +maxdegree = 14 + +# Make data set. +x = np.linspace(-3, 3, n).reshape(-1, 1) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape) +error = np.zeros(maxdegree) +bias = np.zeros(maxdegree) +variance = np.zeros(maxdegree) +polydegree = np.zeros(maxdegree) +X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2) + +from sklearn.preprocessing import StandardScaler +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) + +for degree in range(maxdegree): + model = DecisionTreeRegressor(max_depth=2) + y_pred = np.empty((y_test.shape[0], n_boostraps)) + for i in range(n_boostraps): + x_, y_ = resample(X_train_scaled, y_train) + model.fit(x_, y_) + y_pred[:, i] = model.predict(X_test_scaled).ravel() + + polydegree[degree] = degree + error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) ) + bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 ) + variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) ) + print('Polynomial degree:', degree) + print('Error:', error[degree]) + print('Bias^2:', bias[degree]) + print('Var:', variance[degree]) + print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree])) + +plt.plot(polydegree, error, label='Error') +plt.plot(polydegree, bias, label='bias') +plt.plot(polydegree, variance, label='Variance') +plt.legend() +plt.show() + + + diff --git a/doc/src/SupportVMachines/Programs/cancer.py b/doc/src/SupportVMachines/Programs/cancer.py new file mode 100644 index 000000000..7cc3a8984 --- /dev/null +++ b/doc/src/SupportVMachines/Programs/cancer.py @@ -0,0 +1,41 @@ +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.datasets import load_breast_cancer +from sklearn.svm import SVC +from sklearn.linear_model import LogisticRegression +from sklearn.tree import DecisionTreeClassifier + +# Load the data +cancer = load_breast_cancer() + +X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0) +print(X_train.shape) +print(X_test.shape) +# Logistic Regression +logreg = LogisticRegression(solver='lbfgs') +logreg.fit(X_train, y_train) +print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test))) +# Support vector machine +svm = SVC(gamma='auto', C=100) +svm.fit(X_train, y_train) +print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test))) +# Decision Trees +deep_tree_clf = DecisionTreeClassifier(max_depth=None) +deep_tree_clf.fit(X_train, y_train) +print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test))) +#now scale the data +from sklearn.preprocessing import StandardScaler +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) +# Logistic Regression +logreg.fit(X_train_scaled, y_train) +print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test))) +# Support Vector Machine +svm.fit(X_train_scaled, y_train) +print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test))) +# Decision Trees +deep_tree_clf.fit(X_train_scaled, y_train) +print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test))) diff --git a/doc/src/SupportVMachines/Programs/cart.py b/doc/src/SupportVMachines/Programs/cart.py new file mode 100644 index 000000000..afcfa34c3 --- /dev/null +++ b/doc/src/SupportVMachines/Programs/cart.py @@ -0,0 +1,172 @@ +# CART on the Bank Note dataset +from random import seed +from random import randrange +from csv import reader + +# Load a CSV file +def load_csv(filename): + file = open(filename, "rb") + lines = reader(file) + dataset = list(lines) + return dataset + +# Convert string column to float +def str_column_to_float(dataset, column): + for row in dataset: + row[column] = float(row[column].strip()) + +# Split a dataset into k folds +def cross_validation_split(dataset, n_folds): + dataset_split = list() + dataset_copy = list(dataset) + fold_size = int(len(dataset) / n_folds) + for i in range(n_folds): + fold = list() + while len(fold) < fold_size: + index = randrange(len(dataset_copy)) + fold.append(dataset_copy.pop(index)) + dataset_split.append(fold) + return dataset_split + +# Calculate accuracy percentage +def accuracy_metric(actual, predicted): + correct = 0 + for i in range(len(actual)): + if actual[i] == predicted[i]: + correct += 1 + return correct / float(len(actual)) * 100.0 + +# Evaluate an algorithm using a cross validation split +def evaluate_algorithm(dataset, algorithm, n_folds, *args): + folds = cross_validation_split(dataset, n_folds) + scores = list() + for fold in folds: + train_set = list(folds) + train_set.remove(fold) + train_set = sum(train_set, []) + test_set = list() + for row in fold: + row_copy = list(row) + test_set.append(row_copy) + row_copy[-1] = None + predicted = algorithm(train_set, test_set, *args) + actual = [row[-1] for row in fold] + accuracy = accuracy_metric(actual, predicted) + scores.append(accuracy) + return scores + +# Split a dataset based on an attribute and an attribute value +def test_split(index, value, dataset): + left, right = list(), list() + for row in dataset: + if row[index] < value: + left.append(row) + else: + right.append(row) + return left, right + +# Calculate the Gini index for a split dataset +def gini_index(groups, classes): + # count all samples at split point + n_instances = float(sum([len(group) for group in groups])) + # sum weighted Gini index for each group + gini = 0.0 + for group in groups: + size = float(len(group)) + # avoid divide by zero + if size == 0: + continue + score = 0.0 + # score the group based on the score for each class + for class_val in classes: + p = [row[-1] for row in group].count(class_val) / size + score += p * p + # weight the group score by its relative size + gini += (1.0 - score) * (size / n_instances) + return gini + +# Select the best split point for a dataset +def get_split(dataset): + class_values = list(set(row[-1] for row in dataset)) + b_index, b_value, b_score, b_groups = 999, 999, 999, None + for index in range(len(dataset[0])-1): + for row in dataset: + groups = test_split(index, row[index], dataset) + gini = gini_index(groups, class_values) + if gini < b_score: + b_index, b_value, b_score, b_groups = index, row[index], gini, groups + return {'index':b_index, 'value':b_value, 'groups':b_groups} + +# Create a terminal node value +def to_terminal(group): + outcomes = [row[-1] for row in group] + return max(set(outcomes), key=outcomes.count) + +# Create child splits for a node or make terminal +def split(node, max_depth, min_size, depth): + left, right = node['groups'] + del(node['groups']) + # check for a no split + if not left or not right: + node['left'] = node['right'] = to_terminal(left + right) + return + # check for max depth + if depth >= max_depth: + node['left'], node['right'] = to_terminal(left), to_terminal(right) + return + # process left child + if len(left) <= min_size: + node['left'] = to_terminal(left) + else: + node['left'] = get_split(left) + split(node['left'], max_depth, min_size, depth+1) + # process right child + if len(right) <= min_size: + node['right'] = to_terminal(right) + else: + node['right'] = get_split(right) + split(node['right'], max_depth, min_size, depth+1) + +# Build a decision tree +def build_tree(train, max_depth, min_size): + root = get_split(train) + split(root, max_depth, min_size, 1) + return root + +# Make a prediction with a decision tree +def predict(node, row): + if row[node['index']] < node['value']: + if isinstance(node['left'], dict): + return predict(node['left'], row) + else: + return node['left'] + else: + if isinstance(node['right'], dict): + return predict(node['right'], row) + else: + return node['right'] + +# Classification and Regression Tree Algorithm +def decision_tree(train, test, max_depth, min_size): + tree = build_tree(train, max_depth, min_size) + predictions = list() + for row in test: + prediction = predict(tree, row) + predictions.append(prediction) + return(predictions) + +# Test CART on Bank Note dataset +seed(1) +# load and prepare data +filename = 'DataFiles/rideclass.csv' +dataset = load_csv(filename) +# convert string attributes to integers +for i in range(len(dataset[0])): + str_column_to_float(dataset, i) +# evaluate algorithm +n_folds = 5 +max_depth = 5 +min_size = 10 +scores = evaluate_algorithm(dataset, decision_tree, n_folds, max_depth, min_size) +print('Scores: %s' % scores) +print('Mean Accuracy: %.3f%%' % (sum(scores)/float(len(scores)))) diff --git a/doc/src/SupportVMachines/Programs/decisiontree.py b/doc/src/SupportVMachines/Programs/decisiontree.py new file mode 100644 index 000000000..1142d2d84 --- /dev/null +++ b/doc/src/SupportVMachines/Programs/decisiontree.py @@ -0,0 +1,187 @@ +import re +import math +from collections import deque + +# x is examples in training set +# y is set of attributes +# label is target attributes +# Node is a class which has properties values, childs, and next +# root is top node in the decision tree + +class Node(object): + def __init__(self): + self.value = None + self.next = None + self.childs = None + +# Simple class of Decision Tree +# Aimed for who want to learn Decision Tree, so it is not optimized +class DecisionTree(object): + def __init__(self, sample, attributes, labels): + self.sample = sample + self.attributes = attributes + self.labels = labels + self.labelCodes = None + self.labelCodesCount = None + self.initLabelCodes() + # print(self.labelCodes) + self.root = None + self.entropy = self.getEntropy([x for x in range(len(self.labels))]) + + def initLabelCodes(self): + self.labelCodes = [] + self.labelCodesCount = [] + for l in self.labels: + if l not in self.labelCodes: + self.labelCodes.append(l) + self.labelCodesCount.append(0) + self.labelCodesCount[self.labelCodes.index(l)] += 1 + + def getLabelCodeId(self, sampleId): + return self.labelCodes.index(self.labels[sampleId]) + + def getAttributeValues(self, sampleIds, attributeId): + vals = [] + for sid in sampleIds: + val = self.sample[sid][attributeId] + if val not in vals: + vals.append(val) + # print(vals) + return vals + + def getEntropy(self, sampleIds): + entropy = 0 + labelCount = [0] * len(self.labelCodes) + for sid in sampleIds: + labelCount[self.getLabelCodeId(sid)] += 1 + # print("-ge", labelCount) + for lv in labelCount: + # print(lv) + if lv != 0: + entropy += -lv/len(sampleIds) * math.log(lv/len(sampleIds), 2) + else: + entropy += 0 + return entropy + + def getDominantLabel(self, sampleIds): + labelCodesCount = [0] * len(self.labelCodes) + for sid in sampleIds: + labelCodesCount[self.labelCodes.index(self.labels[sid])] += 1 + return self.labelCodes[labelCodesCount.index(max(labelCodesCount))] + + def getInformationGain(self, sampleIds, attributeId): + gain = self.getEntropy(sampleIds) + attributeVals = [] + attributeValsCount = [] + attributeValsIds = [] + for sid in sampleIds: + val = self.sample[sid][attributeId] + if val not in attributeVals: + attributeVals.append(val) + attributeValsCount.append(0) + attributeValsIds.append([]) + vid = attributeVals.index(val) + attributeValsCount[vid] += 1 + attributeValsIds[vid].append(sid) + # print("-gig", self.attributes[attributeId]) + for vc, vids in zip(attributeValsCount, attributeValsIds): + # print("-gig", vids) + gain -= vc/len(sampleIds) * self.getEntropy(vids) + return gain + + def getAttributeMaxInformationGain(self, sampleIds, attributeIds): + attributesEntropy = [0] * len(attributeIds) + for i, attId in zip(range(len(attributeIds)), attributeIds): + attributesEntropy[i] = self.getInformationGain(sampleIds, attId) + maxId = attributeIds[attributesEntropy.index(max(attributesEntropy))] + return self.attributes[maxId], maxId + + def isSingleLabeled(self, sampleIds): + label = self.labels[sampleIds[0]] + for sid in sampleIds: + if self.labels[sid] != label: + return False + return True + + def getLabel(self, sampleId): + return self.labels[sampleId] + + def id3(self): + sampleIds = [x for x in range(len(self.sample))] + attributeIds = [x for x in range(len(self.attributes))] + self.root = self.id3Recv(sampleIds, attributeIds, self.root) + + def id3Recv(self, sampleIds, attributeIds, root): + root = Node() # Initialize current root + if self.isSingleLabeled(sampleIds): + root.value = self.labels[sampleIds[0]] + return root + # print(attributeIds) + if len(attributeIds) == 0: + root.value = self.getDominantLabel(sampleIds) + return root + bestAttrName, bestAttrId = self.getAttributeMaxInformationGain( + sampleIds, attributeIds) + # print(bestAttrName) + root.value = bestAttrName + root.childs = [] # Create list of children + for value in self.getAttributeValues(sampleIds, bestAttrId): + # print(value) + child = Node() + child.value = value + root.childs.append(child) # Append new child node to current + # root + childSampleIds = [] + for sid in sampleIds: + if self.sample[sid][bestAttrId] == value: + childSampleIds.append(sid) + if len(childSampleIds) == 0: + child.next = self.getDominantLabel(sampleIds) + else: + # print(bestAttrName, bestAttrId) + # print(attributeIds) + if len(attributeIds) > 0 and bestAttrId in attributeIds: + toRemove = attributeIds.index(bestAttrId) + attributeIds.pop(toRemove) + child.next = self.id3Recv( + childSampleIds, attributeIds, child.next) + return root + + def printTree(self): + if self.root: + roots = deque() + roots.append(self.root) + while len(roots) > 0: + root = roots.popleft() + print(root.value) + if root.childs: + for child in root.childs: + print('({})'.format(child.value)) + roots.append(child.next) + elif root.next: + print(root.next) + + +def test(): + f = open('DataFiles/rideclass.csv') + attributes = f.readline().split(',') + attributes = attributes[1:len(attributes)-1] + print(attributes) + sample = f.readlines() + f.close() + for i in range(len(sample)): + sample[i] = re.sub('\d+,', '', sample[i]) + sample[i] = sample[i].strip().split(',') + labels = [] + for s in sample: + labels.append(s.pop()) + # print(sample) + # print(labels) + decisionTree = DecisionTree(sample, attributes, labels) + print("System entropy {}".format(decisionTree.entropy)) + decisionTree.id3() + decisionTree.printTree() + + +if __name__ == '__main__': + test() diff --git a/doc/src/SupportVMachines/Programs/dtcancer.py b/doc/src/SupportVMachines/Programs/dtcancer.py new file mode 100644 index 000000000..ee4e677d2 --- /dev/null +++ b/doc/src/SupportVMachines/Programs/dtcancer.py @@ -0,0 +1,57 @@ +import os +from sklearn.datasets import load_breast_cancer +from sklearn.tree import DecisionTreeClassifier +from sklearn.model_selection import train_test_split +from sklearn.metrics import confusion_matrix +from sklearn.tree import export_graphviz + +from IPython.display import Image +from pydot import graph_from_dot_data +import pandas as pd +import numpy as np + + +# Where to save the figures and data files +PROJECT_ROOT_DIR = "Results" +FIGURE_ID = "Results/FigureFiles" +DATA_ID = "DataFiles/" + +if not os.path.exists(PROJECT_ROOT_DIR): + os.mkdir(PROJECT_ROOT_DIR) + +if not os.path.exists(FIGURE_ID): + os.makedirs(FIGURE_ID) + +if not os.path.exists(DATA_ID): + os.makedirs(DATA_ID) + +def image_path(fig_id): + return os.path.join(FIGURE_ID, fig_id) + +def data_path(dat_id): + return os.path.join(DATA_ID, dat_id) + +def save_fig(fig_id): + plt.savefig(image_path(fig_id) + ".png", format='png') + + +cancer = load_breast_cancer() +X = pd.DataFrame(cancer.data, columns=cancer.feature_names) +print(X) +y = pd.Categorical.from_codes(cancer.target, cancer.target_names) +y = pd.get_dummies(y) +print(y) +X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1) +tree_clf = DecisionTreeClassifier(max_depth=5) +tree_clf.fit(X_train, y_train) + +export_graphviz( + tree_clf, + out_file="DataFiles/cancer.dot", + feature_names=cancer.feature_names, + class_names=cancer.target_names, + rounded=True, + filled=True +) +cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png' +os.system(cmd) diff --git a/doc/src/SupportVMachines/Programs/gdclas.py b/doc/src/SupportVMachines/Programs/gdclas.py new file mode 100644 index 000000000..c7e81b05e --- /dev/null +++ b/doc/src/SupportVMachines/Programs/gdclas.py @@ -0,0 +1,37 @@ +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.datasets import load_breast_cancer +import scikitplot as skplt +from sklearn.ensemble import GradientBoostingClassifier +from sklearn.model_selection import cross_validate + +# Load the data +cancer = load_breast_cancer() + +X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0) +print(X_train.shape) +print(X_test.shape) +#now scale the data +from sklearn.preprocessing import StandardScaler +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) + +gd_clf = GradientBoostingClassifier(max_depth=3, n_estimators=100, learning_rate=1.0) +gd_clf.fit(X_train_scaled, y_train) +#Cross validation +accuracy = cross_validate(gd_clf,X_test_scaled,y_test,cv=10)['test_score'] +print(accuracy) +print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(gd_clf.score(X_test_scaled,y_test))) + +import scikitplot as skplt +y_pred = gd_clf.predict(X_test_scaled) +skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True) +plt.show() +y_probas = gd_clf.predict_proba(X_test_scaled) +skplt.metrics.plot_roc(y_test, y_probas) +plt.show() +skplt.metrics.plot_cumulative_gain(y_test, y_probas) +plt.show() diff --git a/doc/src/SupportVMachines/Programs/gdreg.py b/doc/src/SupportVMachines/Programs/gdreg.py new file mode 100644 index 000000000..d6c1601c7 --- /dev/null +++ b/doc/src/SupportVMachines/Programs/gdreg.py @@ -0,0 +1,48 @@ +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.ensemble import GradientBoostingRegressor +from sklearn.preprocessing import StandardScaler +import scikitplot as skplt +from sklearn.metrics import mean_squared_error + +n = 1000 +maxdegree = 6 + +# Make data set. +x = np.linspace(-3, 3, n).reshape(-1, 1) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape) + +error = np.zeros(maxdegree) +bias = np.zeros(maxdegree) +variance = np.zeros(maxdegree) +polydegree = np.zeros(maxdegree) +X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2) +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) + +for degree in range(1,maxdegree): + model = GradientBoostingRegressor(max_depth=degree, n_estimators=100, learning_rate=1.0) + model.fit(X_train_scaled,y_train) + y_pred = model.predict(X_test_scaled) + polydegree[degree] = degree + error[degree] = np.mean( np.mean((y_test - y_pred)**2) ) + bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 ) + variance[degree] = np.mean( np.var(y_pred) ) + print('Max depth:', degree) + print('Error:', error[degree]) + print('Bias^2:', bias[degree]) + print('Var:', variance[degree]) + print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree])) + +plt.xlim(1,maxdegree-1) +plt.plot(polydegree, error, label='Error') +plt.plot(polydegree, bias, label='bias') +plt.plot(polydegree, variance, label='Variance') +plt.legend() +plt.show() + + + diff --git a/doc/src/SupportVMachines/Programs/gdreg.py~ b/doc/src/SupportVMachines/Programs/gdreg.py~ new file mode 100644 index 000000000..1fc003877 --- /dev/null +++ b/doc/src/SupportVMachines/Programs/gdreg.py~ @@ -0,0 +1,48 @@ +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.ensemble import GradientBoostingRegressor +from sklearn.preprocessing import StandardScaler +import scikitplot as skplt +from sklearn.metrics import mean_squared_error + +n = 1000 +maxdegree = 6 + +# Make data set. +x = np.linspace(-3, 3, n).reshape(-1, 1) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape) + +error = np.zeros(maxdegree) +bias = np.zeros(maxdegree) +variance = np.zeros(maxdegree) +polydegree = np.zeros(maxdegree) +X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2) +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) + +for degree in range(1,maxdegree): + model = GradientBoostingRegressor(max_depth=degree, n_estimators=3, learning_rate=1.0) + model.fit(X_train_scaled,y_train) + y_pred = model.predict(X_test_scaled) + polydegree[degree] = degree + error[degree] = np.mean( np.mean((y_test - y_pred)**2) ) + bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 ) + variance[degree] = np.mean( np.var(y_pred) ) + print('Max depth:', degree) + print('Error:', error[degree]) + print('Bias^2:', bias[degree]) + print('Var:', variance[degree]) + print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree])) + +plt.xlim(1,maxdegree-1) +plt.plot(polydegree, error, label='Error') +plt.plot(polydegree, bias, label='bias') +plt.plot(polydegree, variance, label='Variance') +plt.legend() +plt.show() + + + diff --git a/doc/src/SupportVMachines/Programs/gini.py b/doc/src/SupportVMachines/Programs/gini.py new file mode 100644 index 000000000..cf4509b9c --- /dev/null +++ b/doc/src/SupportVMachines/Programs/gini.py @@ -0,0 +1,74 @@ +# Split a dataset based on an attribute and an attribute value +def test_split(index, value, dataset): + left, right = list(), list() + for row in dataset: + if row[index] < value: + left.append(row) + else: + right.append(row) + return left, right + +# Calculate the Gini index for a split dataset +def gini_index(groups, classes): + # count all samples at split point + n_instances = float(sum([len(group) for group in groups])) + # sum weighted Gini index for each group + gini = 0.0 + for group in groups: + size = float(len(group)) + # avoid divide by zero + if size == 0: + continue + score = 0.0 + # score the group based on the score for each class + for class_val in classes: + p = [row[-1] for row in group].count(class_val) / size + score += p * p + # weight the group score by its relative size + gini += (1.0 - score) * (size / n_instances) + return gini + +# Select the best split point for a dataset +def get_split(dataset): + class_values = list(set(row[-1] for row in dataset)) + b_index, b_value, b_score, b_groups = 999, 999, 999, None + for index in range(len(dataset[0])-1): + for row in dataset: + groups = test_split(index, row[index], dataset) + gini = gini_index(groups, class_values) + print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini)) + if gini < b_score: + b_index, b_value, b_score, b_groups = index, row[index], gini, groups + return {'index':b_index, 'value':b_value, 'groups':b_groups} + +""" +dataset = [[2.771244718,1.784783929,0], + [1.728571309,1.169761413,0], + [3.678319846,2.81281357,0], + [3.961043357,2.61995032,0], + [2.999208922,2.209014212,0], + [7.497545867,3.162953546,1], + [9.00220326,3.339047188,1], + [7.444542326,0.476683375,1], + [10.12493903,3.234550982,1], + [6.642287351,3.319983761,1]] +""" +dataset = [[0,0,0,0,0], + [0,0,0,1,1], + [1,0,0,0,1], + [2,1,0,0,1], + [2,2,1,0,1], + [2,2,1,1,0], + [1,2,1,1,1], + [0,1,0,0,0], + [0,2,1,0,1], + [2,1,1,0,1], + [0,1,1,1,1], + [1,1,0,1,1], + [1,0,1,0,1], + [2,1,0,1,0]] + +split = get_split(dataset) +print('Split: [X%d < %.3f]' % ((split['index']+1), split['value'])) + + diff --git a/doc/src/SupportVMachines/Programs/moons.py b/doc/src/SupportVMachines/Programs/moons.py new file mode 100644 index 000000000..bde45069e --- /dev/null +++ b/doc/src/SupportVMachines/Programs/moons.py @@ -0,0 +1,28 @@ +# Common imports +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.tree import DecisionTreeClassifier +from sklearn.datasets import make_moons +from sklearn.tree import export_graphviz +from pydot import graph_from_dot_data +import pandas as pd + + +np.random.seed(42) +X, y = make_moons(n_samples=100, noise=0.25, random_state=53) +X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0) +tree_clf = DecisionTreeClassifier(max_depth=5) +tree_clf.fit(X_train, y_train) + +export_graphviz( + tree_clf, + out_file="moons.dot", +# feature_names=tree_clf.feature_names, +# class_names=tree_clf.target_names, + rounded=True, + filled=True +) + + + + diff --git a/doc/src/SupportVMachines/Programs/rfbootclassify.py b/doc/src/SupportVMachines/Programs/rfbootclassify.py new file mode 100644 index 000000000..6bd25a532 --- /dev/null +++ b/doc/src/SupportVMachines/Programs/rfbootclassify.py @@ -0,0 +1,54 @@ + +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.utils import resample +from sklearn.tree import DecisionTreeRegressor + +n = 100 +n_boostraps = 100 +maxdepth = 8 + +# Make data set. +x = np.linspace(-3, 3, n).reshape(-1, 1) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape) +error = np.zeros(maxdepth) +bias = np.zeros(maxdepth) +variance = np.zeros(maxdepth) +polydegree = np.zeros(maxdepth) +X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2) + +from sklearn.preprocessing import StandardScaler +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) + +for degree in range(1,maxdepth): + model = DecisionTreeRegressor(max_depth=degree) + y_pred = np.empty((y_test.shape[0], n_boostraps)) + for i in range(n_boostraps): + x_, y_ = resample(X_train_scaled, y_train) + model.fit(x_, y_) + y_pred[:, i] = model.predict(X_test_scaled)#.ravel() + + polydegree[degree] = degree + error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) ) + bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 ) + variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) ) + print('Polynomial degree:', degree) + print('Error:', error[degree]) + print('Bias^2:', bias[degree]) + print('Var:', variance[degree]) + print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree])) + +plt.xlim(1,maxdepth) +plt.plot(polydegree, error, label='Error') +plt.plot(polydegree, bias, label='bias') +plt.plot(polydegree, variance, label='Variance') +plt.legend() +plt.show() + + + diff --git a/doc/src/SupportVMachines/Programs/rfbootclassify.py~ b/doc/src/SupportVMachines/Programs/rfbootclassify.py~ new file mode 100644 index 000000000..84994c400 --- /dev/null +++ b/doc/src/SupportVMachines/Programs/rfbootclassify.py~ @@ -0,0 +1,54 @@ + +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.utils import resample +from sklearn.tree import DecisionTreeRegressor + +n = 100 +n_boostraps = 100 +maxdepth = 8 + +# Make data set. +x = np.linspace(-3, 3, n).reshape(-1, 1) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape) +error = np.zeros(maxdepth) +bias = np.zeros(maxdepth) +variance = np.zeros(maxdepth) +polydegree = np.zeros(maxdepth) +X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2) + +from sklearn.preprocessing import StandardScaler +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) + +for degree in range(1,maxdepth): + model = DecisionTreeRegressor(max_depth=degree) + y_pred = np.empty((y_test.shape[0], n_boostraps)) + for i in range(n_boostraps): + x_, y_ = resample(X_train_scaled, y_train) + model.fit(x_, y_) + y_pred[:, i] = model.predict(X_test_scaled)#.ravel() + + polydegree[degree] = degree + error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) ) + bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 ) + variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) ) + print('Polynomial degree:', degree) + print('Error:', error[degree]) + print('Bias^2:', bias[degree]) + print('Var:', variance[degree]) + print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree])) + +plt.xlim(1,maxdegree) +plt.plot(polydegree, error, label='Error') +plt.plot(polydegree, bias, label='bias') +plt.plot(polydegree, variance, label='Variance') +plt.legend() +plt.show() + + + diff --git a/doc/src/SupportVMachines/Programs/rfbootstrap.py b/doc/src/SupportVMachines/Programs/rfbootstrap.py new file mode 100644 index 000000000..db31345b5 --- /dev/null +++ b/doc/src/SupportVMachines/Programs/rfbootstrap.py @@ -0,0 +1,55 @@ + +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.utils import resample +from sklearn.ensemble import RandomForestRegressor + +np.random.seed(2018) + +n = 500 +n_boostraps = 100 +maxdegree = 14 + +# Make data set. +x = np.linspace(-3, 3, n).reshape(-1, 1) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape) + +error = np.zeros(maxdegree) +bias = np.zeros(maxdegree) +variance = np.zeros(maxdegree) +polydegree = np.zeros(maxdegree) +X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2) + +from sklearn.preprocessing import StandardScaler +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) + +for degree in range(maxdegree): + model = RandomForestRegressor() + y_pred = np.empty((y_test.shape[0], n_boostraps)) + for i in range(n_boostraps): + x_, y_ = resample(X_train_scaled, y_train) + model.fit(x_, y_.ravel()) + y_pred[:, i] = model.predict(X_test_scaled) + + polydegree[degree] = degree + error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) ) + bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 ) + variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) ) + print('Polynomial degree:', degree) + print('Error:', error[degree]) + print('Bias^2:', bias[degree]) + print('Var:', variance[degree]) + print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree])) + +plt.plot(polydegree, error, label='Error') +plt.plot(polydegree, bias, label='bias') +plt.plot(polydegree, variance, label='Variance') +plt.legend() +plt.show() + + diff --git a/doc/src/SupportVMachines/Programs/rfbootstrap.py~ b/doc/src/SupportVMachines/Programs/rfbootstrap.py~ new file mode 100644 index 000000000..96f81e3df --- /dev/null +++ b/doc/src/SupportVMachines/Programs/rfbootstrap.py~ @@ -0,0 +1,54 @@ + +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.utils import resample +from sklearn.ensemble import RandomForestRegressor + +np.random.seed(2018) + +n = 40 +n_boostraps = 100 +maxdegree = 14 + +# Make data set. +x = np.linspace(-3, 3, n).reshape(-1, 1) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape) +error = np.zeros(maxdegree) +bias = np.zeros(maxdegree) +variance = np.zeros(maxdegree) +polydegree = np.zeros(maxdegree) +X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2) + +from sklearn.preprocessing import StandardScaler +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) + +for degree in range(maxdegree): + model = RandomForestRegressor() + y_pred = np.empty((y_test.shape[0], n_boostraps)) + for i in range(n_boostraps): + x_, y_ = resample(X_train_scaled, y_train).ravel() + model.fit(x_, y_) + y_pred[:, i] = model.predict(X_test_scaled).ravel() + + polydegree[degree] = degree + error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) ) + bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 ) + variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) ) + print('Polynomial degree:', degree) + print('Error:', error[degree]) + print('Bias^2:', bias[degree]) + print('Var:', variance[degree]) + print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree])) + +plt.plot(polydegree, error, label='Error') +plt.plot(polydegree, bias, label='bias') +plt.plot(polydegree, variance, label='Variance') +plt.legend() +plt.show() + + diff --git a/doc/src/SupportVMachines/Programs/rfcancer.py b/doc/src/SupportVMachines/Programs/rfcancer.py new file mode 100644 index 000000000..d507268cb --- /dev/null +++ b/doc/src/SupportVMachines/Programs/rfcancer.py @@ -0,0 +1,39 @@ +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.datasets import load_breast_cancer +from sklearn.tree import DecisionTreeClassifier +from sklearn.ensemble import RandomForestClassifier +from sklearn.preprocessing import LabelEncoder +from sklearn.model_selection import cross_validate +import scikitplot as skplt + +# Load the data +cancer = load_breast_cancer() + +X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0) +print(X_train.shape) +print(X_test.shape) +#now scale the data +from sklearn.preprocessing import StandardScaler +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) +# Data set not specificied +#Instantiate the model with 100 trees and entropy as splitting criteria +Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy") +Random_Forest_model.fit(X_train_scaled, y_train) +#Cross validation +accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score'] +print(accuracy) +print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test))) + +y_pred = Random_Forest_model.predict(X_test_scaled) +skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True) +plt.show() +y_probas = Random_Forest_model.predict_proba(X_test_scaled) +skplt.metrics.plot_roc(y_test, y_probas) +plt.show() +skplt.metrics.plot_cumulative_gain(y_test, y_probas) +plt.show() diff --git a/doc/src/SupportVMachines/Programs/ride.py b/doc/src/SupportVMachines/Programs/ride.py new file mode 100644 index 000000000..90567fd10 --- /dev/null +++ b/doc/src/SupportVMachines/Programs/ride.py @@ -0,0 +1,68 @@ +# Common imports +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.tree import DecisionTreeClassifier +from sklearn.model_selection import train_test_split +from sklearn.tree import export_graphviz +from sklearn.preprocessing import StandardScaler, OneHotEncoder +from sklearn.compose import ColumnTransformer +from IPython.display import Image +from pydot import graph_from_dot_data +import os + +# Where to save the figures and data files +PROJECT_ROOT_DIR = "Results" +FIGURE_ID = "Results/FigureFiles" +DATA_ID = "DataFiles/" + +if not os.path.exists(PROJECT_ROOT_DIR): + os.mkdir(PROJECT_ROOT_DIR) + +if not os.path.exists(FIGURE_ID): + os.makedirs(FIGURE_ID) + +if not os.path.exists(DATA_ID): + os.makedirs(DATA_ID) + +def image_path(fig_id): + return os.path.join(FIGURE_ID, fig_id) + +def data_path(dat_id): + return os.path.join(DATA_ID, dat_id) + +def save_fig(fig_id): + plt.savefig(image_path(fig_id) + ".png", format='png') + +infile = open(data_path("rideclass.csv"),'r') + +# Read the experimental data with Pandas +from IPython.display import display +ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride')) +ridedata = pd.DataFrame(ridedata) + +# Features and targets +X = ridedata.loc[:, ridedata.columns != 'Ride'].values +y = ridedata.loc[:, ridedata.columns == 'Ride'].values + +# Create the encoder. +encoder = OneHotEncoder(handle_unknown="ignore") +# Assume for simplicity all features are categorical. +encoder.fit(X) +# Apply the encoder. +X = encoder.transform(X) +print(X) +# Then do a Classification tree +tree_clf = DecisionTreeClassifier(max_depth=2) +tree_clf.fit(X, y) +print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y))) +#transfer to a decision tree graph +export_graphviz( + tree_clf, + out_file="DataFiles/ride.dot", + rounded=True, + filled=True +) +cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png' +os.system(cmd) + diff --git a/doc/src/SupportVMachines/Programs/xgcancer.py b/doc/src/SupportVMachines/Programs/xgcancer.py new file mode 100644 index 000000000..bad114975 --- /dev/null +++ b/doc/src/SupportVMachines/Programs/xgcancer.py @@ -0,0 +1,46 @@ +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.datasets import load_breast_cancer +from sklearn.preprocessing import LabelEncoder +from sklearn.model_selection import cross_validate +import scikitplot as skplt +import xgboost as xgb +# Load the data +cancer = load_breast_cancer() + +X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0) +print(X_train.shape) +print(X_test.shape) +#now scale the data +from sklearn.preprocessing import StandardScaler +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) + +xg_clf = xgb.XGBClassifier(max_depth = 4, n_estimators = 200) +xg_clf.fit(X_train_scaled,y_train) + +y_test = xg_clf.predict(X_test_scaled) + +print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(xg_clf.score(X_test_scaled,y_test))) + +import scikitplot as skplt +y_pred = xg_clf.predict(X_test_scaled) +skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True) +plt.show() +y_probas = xg_clf.predict_proba(X_test_scaled) +skplt.metrics.plot_roc(y_test, y_probas) +plt.show() +skplt.metrics.plot_cumulative_gain(y_test, y_probas) +plt.show() + + +xgb.plot_tree(xg_clf,num_trees=0) +plt.rcParams['figure.figsize'] = [50, 10] +plt.show() + +xgb.plot_importance(xg_clf) +plt.rcParams['figure.figsize'] = [5, 5] +plt.show() diff --git a/doc/src/SupportVMachines/Programs/xgcancer.py~ b/doc/src/SupportVMachines/Programs/xgcancer.py~ new file mode 100644 index 000000000..9f226f064 --- /dev/null +++ b/doc/src/SupportVMachines/Programs/xgcancer.py~ @@ -0,0 +1,41 @@ +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.datasets import load_breast_cancer +from sklearn.tree import DecisionTreeClassifier +from sklearn.ensemble import RandomForestClassifier +from sklearn.preprocessing import LabelEncoder +from sklearn.model_selection import cross_validate +import scikitplot as skplt + +# Load the data +cancer = load_breast_cancer() + +X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0) +print(X_train.shape) +print(X_test.shape) +#now scale the data +from sklearn.preprocessing import StandardScaler +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) +# Data set not specificied +#Instantiate the model with 100 trees and entropy as splitting criteria +Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy") +Random_Forest_model.fit(X_train_scaled, y_train) +#Cross validation +accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score'] +print(accuracy) +print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test))) + + + +y_pred = Random_Forest_model.predict(X_test_scaled) +skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True) +plt.show() +y_probas = Random_Forest_model.predict_proba(X_test_scaled) +skplt.metrics.plot_roc(y_test, y_probas) +plt.show() +skplt.metrics.plot_cumulative_gain(y_test, y_probas) +plt.show() diff --git a/doc/src/SupportVMachines/Programs/xgregressor.py b/doc/src/SupportVMachines/Programs/xgregressor.py new file mode 100644 index 000000000..6a58b6bf1 --- /dev/null +++ b/doc/src/SupportVMachines/Programs/xgregressor.py @@ -0,0 +1,65 @@ +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +import xgboost as xgb +from sklearn.preprocessing import StandardScaler +import scikitplot as skplt +from sklearn.metrics import mean_squared_error + + +def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True): + x1s = np.linspace(axes[0], axes[1], 100) + x2s = np.linspace(axes[2], axes[3], 100) + x1, x2 = np.meshgrid(x1s, x2s) + X_new = np.c_[x1.ravel(), x2.ravel()] + y_pred = clf.predict(X_new).reshape(x1.shape) + custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0']) + plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap) + if contour: + custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50']) + plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8) + plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha) + plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha) + plt.axis(axes) + plt.xlabel(r"$x_1$", fontsize=18) + plt.ylabel(r"$x_2$", fontsize=18, rotation=0) + +n = 500 +maxdegree = 8 + +# Make data set. +x = np.linspace(-3, 3, n).reshape(-1, 1) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape) + +error = np.zeros(maxdegree) +bias = np.zeros(maxdegree) +variance = np.zeros(maxdegree) +polydegree = np.zeros(maxdegree) +X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2) +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) + +for degree in range(maxdegree): + model = xgb.XGBRegressor(objective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1, + max_depth = degree, alpha = 10, n_estimators = 10) + model.fit(X_train_scaled,y_train) + y_pred = model.predict(X_test_scaled) + polydegree[degree] = degree + error[degree] = np.mean( np.mean((y_test - y_pred)**2) ) + bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 ) + variance[degree] = np.mean( np.var(y_pred) ) + print('Polynomial degree:', degree) + print('Error:', error[degree]) + print('Bias^2:', bias[degree]) + print('Var:', variance[degree]) + print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree])) + +plt.xlim(1,maxdegree-1) +plt.plot(polydegree, error, label='Error') +plt.plot(polydegree, bias, label='bias') +plt.plot(polydegree, variance, label='Variance') +plt.legend() +plt.show() + diff --git a/doc/src/SupportVMachines/Programs/xgregressor.py~ b/doc/src/SupportVMachines/Programs/xgregressor.py~ new file mode 100644 index 000000000..6c9b2a16d --- /dev/null +++ b/doc/src/SupportVMachines/Programs/xgregressor.py~ @@ -0,0 +1,47 @@ +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +import xgboost as xgb +from sklearn.preprocessing import StandardScaler +import scikitplot as skplt +from sklearn.metrics import mean_squared_error + +n = 500 +n_boostraps = 100 +maxdegree = 8 + +# Make data set. +x = np.linspace(-3, 3, n).reshape(-1, 1) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape) + +error = np.zeros(maxdegree) +bias = np.zeros(maxdegree) +variance = np.zeros(maxdegree) +polydegree = np.zeros(maxdegree) +X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2) +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) + +for degree in range(maxdegree): + model = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1, + max_depth = maxdegree, alpha = 10, n_estimators = 10) + model.fit(X_train_scaled,y_train) + y_pred = model.predict(X_test_scaled) + polydegree[degree] = degree + error[degree] = np.mean( np.mean((y_test - y_pred)**2) ) + bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 ) + variance[degree] = np.mean( np.var(y_pred) ) + print('Polynomial degree:', degree) + print('Error:', error[degree]) + print('Bias^2:', bias[degree]) + print('Var:', variance[degree]) + print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree])) + +plt.plot(polydegree, error, label='Error') +plt.plot(polydegree, bias, label='bias') +plt.plot(polydegree, variance, label='Variance') +plt.legend() +plt.show() +