added codes
This commit is contained in:
mhjensen
@@ -0,0 +1,300 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Mon Feb 17 10:00:03 2020
|
||||
|
||||
@author: Bharat Mishra, Patricia Perez-Martin, Pinelopi Christodoulou
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.datasets import load_breast_cancer
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
|
||||
# close all previous images
|
||||
plt.close('all')
|
||||
|
||||
# ignore warnings
|
||||
import warnings
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
# ensure the same random numbers appear every time
|
||||
np.random.seed(0)
|
||||
|
||||
# download breast cancer dataset
|
||||
cancer = load_breast_cancer()
|
||||
|
||||
# define inputs and labels
|
||||
inputs = cancer.data
|
||||
outputs = cancer.target #Malignant or bening
|
||||
labels = cancer.feature_names[0:30]
|
||||
|
||||
|
||||
print('The content of the breast cancer dataset is:')
|
||||
print('-------------------------')
|
||||
print("inputs = " + str(inputs.shape))
|
||||
print("outputs = " + str(outputs.shape))
|
||||
print("labels = "+ str(labels.shape))
|
||||
|
||||
n_inputs = len(inputs)
|
||||
|
||||
#%% VISUALIZATION
|
||||
|
||||
X = inputs
|
||||
y = outputs
|
||||
|
||||
plt.figure()
|
||||
plt.scatter(X[:,0], X[:,2], s=40, c=y, cmap=plt.cm.Spectral)
|
||||
plt.xlabel('Mean radius')
|
||||
plt.ylabel('Mean perimeter')
|
||||
plt.show()
|
||||
|
||||
plt.figure()
|
||||
plt.scatter(X[:,5], X[:,6], s=40, c=y, cmap=plt.cm.Spectral)
|
||||
plt.xlabel('Mean compactness')
|
||||
plt.ylabel('Mean concavity')
|
||||
plt.show()
|
||||
|
||||
|
||||
plt.figure()
|
||||
plt.scatter(X[:,0], X[:,1], s=40, c=y, cmap=plt.cm.Spectral)
|
||||
plt.xlabel('Mean radius')
|
||||
plt.ylabel('Mean texture')
|
||||
plt.show()
|
||||
|
||||
plt.figure()
|
||||
plt.scatter(X[:,2], X[:,1], s=40, c=y, cmap=plt.cm.Spectral)
|
||||
plt.xlabel('Mean perimeter')
|
||||
plt.ylabel('Mean compactness')
|
||||
plt.show()
|
||||
|
||||
|
||||
# %% COVARIANCE AND CORRELATION
|
||||
|
||||
import pandas as pd
|
||||
import seaborn as sns
|
||||
|
||||
# Making a data frame
|
||||
|
||||
meanpd = pd.DataFrame(X[:,0:10],columns=labels[0:10])
|
||||
corr = meanpd.corr().round(1) # Compute pairwise correlation of columns, excluding NA/null values.
|
||||
|
||||
# use the heatmap function from seaborn to plot the correlation matrix
|
||||
plt.figure()
|
||||
sns.heatmap(corr, cbar = True, annot=False,
|
||||
xticklabels= labels[0:10], yticklabels= labels[0:10],
|
||||
cmap= 'YlOrRd')
|
||||
|
||||
|
||||
X_t = X[ : , 1:3]
|
||||
|
||||
clf = LogisticRegression()
|
||||
clf.fit(X_t, y)
|
||||
|
||||
|
||||
# Set min and max values and give it some padding
|
||||
x_min, x_max = X_t[:, 1].min() - .5, X_t[:, 1].max() + .5
|
||||
y_min, y_max = X_t[:, 0].min() - .5, X_t[:, 0].max() + .5
|
||||
h = 0.01
|
||||
# Generate a grid of points with distance h between them
|
||||
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
|
||||
# Predict the function value for the whole gid
|
||||
Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
|
||||
#Z = pred_func(np.c_[xx.ravel(), yy.ravel()])
|
||||
Z = Z.reshape(xx.shape)
|
||||
|
||||
# Plot the contour and training examples
|
||||
plt.figure()
|
||||
plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral)
|
||||
plt.scatter(X[:, 2], X[:, 1], c=y, cmap=plt.cm.Spectral)
|
||||
plt.xlabel('Mean perimeter')
|
||||
plt.ylabel('Mean texture')
|
||||
plt.title('Logistic Regression')
|
||||
plt.show()
|
||||
|
||||
# %% TRAIN AND TEST DATASET
|
||||
|
||||
# Set up training data: from scikit-learn library
|
||||
train_size = 0.9
|
||||
test_size = 1 - train_size
|
||||
|
||||
X_train, X_test, y_train, y_test = train_test_split(inputs, outputs, train_size=train_size,
|
||||
test_size=test_size)
|
||||
|
||||
# %% LOGISTIC REGRESSION and ACCURACY
|
||||
|
||||
print('----------------------')
|
||||
print('LOGISTIC REGRESSION')
|
||||
print('----------------------')
|
||||
logreg = LogisticRegression()
|
||||
logreg.fit(X_train, y_train)
|
||||
print("Train set accuracy with Logistic Regression:: {:.2f}".format(logreg.score(X_train,y_train)))
|
||||
print("Test set accuracy with Logistic Regression:: {:.2f}".format(logreg.score(X_test,y_test)))
|
||||
|
||||
# Scale data
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
scaler = StandardScaler()
|
||||
scaler.fit(X_train)
|
||||
X_train_scaled = scaler.transform(X_train)
|
||||
X_test_scaled = scaler.transform(X_test)
|
||||
logreg.fit(X_train_scaled, y_train)
|
||||
print("Train set accuracy Logistic Regression scaled data: {:.2f}".format(logreg.score(X_train_scaled,y_train)))
|
||||
print("Test set accuracy scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
# %% CROSS VALIDATION FROM SCIKIT-LEARN
|
||||
from sklearn.linear_model import LogisticRegressionCV
|
||||
|
||||
print('----------------------')
|
||||
print('LOGISTIC REGRESSION with CROSS VALIDATION 5-KFold')
|
||||
print('----------------------')
|
||||
logreg = LogisticRegressionCV()
|
||||
logreg.fit(X_train, y_train)
|
||||
print("Train set accuracy with Logistic Regression, CV:: {:.2f}".format(logreg.score(X_train,y_train)))
|
||||
print("Test set accuracy with Logistic Regression, CV:: {:.2f}".format(logreg.score(X_test,y_test)))
|
||||
|
||||
# Scale data
|
||||
|
||||
scaler.fit(X_train)
|
||||
X_train_scaled = scaler.transform(X_train)
|
||||
X_test_scaled = scaler.transform(X_test)
|
||||
logreg.fit(X_train_scaled, y_train)
|
||||
print("Train set accuracy Logistic Regression scaled data: {:.2f}".format(logreg.score(X_train_scaled,y_train)))
|
||||
print("Test set accuracy scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
|
||||
|
||||
# %% CROSS VALIDATION: OUR OWN CODE
|
||||
|
||||
"""Implement cross-validation framework (only on liblinear solver)"""
|
||||
|
||||
#Initiate k-fold instance for implementing manual cross-validation using KFold
|
||||
|
||||
import seaborn as sns
|
||||
from sklearn.model_selection import KFold
|
||||
from sklearn.linear_model import LogisticRegression as LogReg
|
||||
import os
|
||||
|
||||
"""Generate training and testing datasets"""
|
||||
|
||||
x=inputs
|
||||
|
||||
#Select features relevant to classification (texture,perimeter,compactness and symmetery)
|
||||
#and add to input matrix
|
||||
|
||||
temp1=np.reshape(inputs[:,1],(len(inputs[:,1]),1))
|
||||
temp2=np.reshape(inputs[:,2],(len(inputs[:,2]),1))
|
||||
X=np.hstack((temp1,temp2))
|
||||
temp=np.reshape(inputs[:,5],(len(inputs[:,5]),1))
|
||||
X=np.hstack((X,temp))
|
||||
temp=np.reshape(inputs[:,8],(len(inputs[:,8]),1))
|
||||
X=np.hstack((X,temp))
|
||||
|
||||
lamda=np.logspace(-5,5,11) #Define array of hyperparameters
|
||||
|
||||
|
||||
"""Implement K-fold cross-validation"""
|
||||
|
||||
k=5
|
||||
kfold=KFold(n_splits=k)
|
||||
|
||||
train_scores=np.zeros((len(lamda),k))
|
||||
test_scores=np.zeros((len(lamda),k))
|
||||
|
||||
for i in range(len(lamda)):
|
||||
j=0
|
||||
for train_inds,test_inds in kfold.split(X):
|
||||
X_train=X[train_inds]
|
||||
y_train=y[train_inds]
|
||||
X_test=X[test_inds]
|
||||
y_test=y[test_inds]
|
||||
|
||||
|
||||
clf=LogReg(C=1.0/lamda[i],random_state=1,verbose=0,max_iter=1E+3,tol=1E-5,solver='liblinear')
|
||||
clf.fit(X_train,y_train)
|
||||
train_scores[i,j]=clf.score(X_train,y_train)
|
||||
test_scores[i,j]=clf.score(X_test,y_test)
|
||||
j+=1
|
||||
|
||||
train_accuracy_cv_kfold=np.mean(train_scores,axis=1)
|
||||
test_accuracy_cv_kfold=np.mean(test_scores,axis=1)
|
||||
|
||||
|
||||
"""Plot results after K-fold cross validation"""
|
||||
|
||||
plt.figure()
|
||||
plt.semilogx(lamda,train_accuracy_cv_kfold,'*-b',label='Training')
|
||||
plt.semilogx(lamda,test_accuracy_cv_kfold,'*-r',label='Test')
|
||||
|
||||
plt.legend()
|
||||
plt.xlabel('Hyperparameters')
|
||||
plt.ylabel('Accuracy')
|
||||
plt.title('Accuracy LogReg (5 k-Fold CV)')
|
||||
plt.show()
|
||||
|
||||
|
||||
# %% DECISION TREES: CLASSIFICATION and ACCURACY
|
||||
|
||||
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.tree import export_graphviz
|
||||
from sklearn.preprocessing import StandardScaler, OneHotEncoder
|
||||
|
||||
|
||||
# Create the encoder.
|
||||
encoder = OneHotEncoder(handle_unknown="ignore")
|
||||
# Assume for simplicity all features are categorical.
|
||||
encoder.fit(X)
|
||||
# Apply the encoder.
|
||||
X = encoder.transform(X)
|
||||
|
||||
|
||||
# Classification tree: with and without scaling (sc)
|
||||
|
||||
DEPTH=np.arange(start=1,stop=11,step=1)
|
||||
test_acc = np.zeros(len(DEPTH))
|
||||
test_acc_sc = np.zeros(len(DEPTH))
|
||||
|
||||
for i in DEPTH:
|
||||
tree_clf = DecisionTreeClassifier(max_depth= i)
|
||||
tree_clf.fit(X_train, y_train)
|
||||
test_acc[i-1] = tree_clf.score(X_test,y_test)
|
||||
print("Decision Tree (No Max depth): {:.2f}".format(DEPTH[i-1]))
|
||||
print(" Test accuracy: {:.2f}".format(test_acc[i-1]))
|
||||
|
||||
export_graphviz(
|
||||
tree_clf,
|
||||
out_file="ride.dot",
|
||||
rounded=True,
|
||||
filled=True
|
||||
)
|
||||
cmd = 'dot -Tpng ride.dot -o DecisionTree_max_depth_{:.2f}.png'.format(DEPTH[i-1])
|
||||
os.system(cmd)
|
||||
|
||||
#PLOT TEST ACCURACY
|
||||
fig,p1=plt.subplots()
|
||||
p1.plot(DEPTH, test_acc, label='Test accuracy')
|
||||
|
||||
|
||||
p1.set_xlabel('Max_depth in Decision Tree')
|
||||
p1.set_ylabel('Accuracy')
|
||||
p1.set_title("Decision Tree Test Accuracy", fontsize=18)
|
||||
p1.legend()
|
||||
|
||||
|
||||
tree_clf = DecisionTreeClassifier(max_depth=None)
|
||||
tree_clf.fit(X_train, y_train)
|
||||
print("Test set accuracy with Decision Tree (No Max depth): {:.2f}".format(tree_clf.score(X_test,y_test)))
|
||||
|
||||
|
||||
# %% RANDOM FOREST and ACCURACY
|
||||
from sklearn.ensemble import RandomForestClassifier
|
||||
from sklearn import metrics
|
||||
|
||||
print('RANDOM FOREST')
|
||||
|
||||
model=RandomForestClassifier(n_estimators= 100)# a simple random forest model
|
||||
model.fit(X_train,y_train)# now fit our model for training data
|
||||
y_pred = model.predict(X_test)# predict for the test data
|
||||
RFtest_acc = metrics.accuracy_score(y_pred,y_test) # to check the accuracy
|
||||
|
||||
|
||||
print("Test set accuracy with RANDOM FOREST: {:.2f}".format(RFtest_acc))
|
||||
Reference in New Issue
Block a user