mhjensen
331 lines
12 KiB
Python
331 lines
12 KiB
Python
# import necessary packages
|
|
import numpy as np
|
|
import matplotlib.pyplot as plt
|
|
from sklearn import datasets
|
|
|
|
|
|
# ensure the same random numbers appear every time
|
|
np.random.seed(0)
|
|
|
|
# display images in notebook
|
|
plt.rcParams['figure.figsize'] = (12,12)
|
|
|
|
|
|
# download MNIST dataset
|
|
digits = datasets.load_digits()
|
|
|
|
# define inputs and labels
|
|
inputs = digits.images
|
|
labels = digits.target
|
|
|
|
# RGB images have a depth of 3
|
|
# our images are grayscale so they should have a depth of 1
|
|
inputs = inputs[:,:,:,np.newaxis]
|
|
|
|
print("inputs = (n_inputs, pixel_width, pixel_height, depth) = " + str(inputs.shape))
|
|
print("labels = (n_inputs) = " + str(labels.shape))
|
|
|
|
|
|
# choose some random images to display
|
|
n_inputs = len(inputs)
|
|
indices = np.arange(n_inputs)
|
|
random_indices = np.random.choice(indices, size=5)
|
|
|
|
for i, image in enumerate(digits.images[random_indices]):
|
|
plt.subplot(1, 5, i+1)
|
|
plt.axis('off')
|
|
plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest')
|
|
plt.title("Label: %d" % digits.target[random_indices[i]])
|
|
plt.show()
|
|
|
|
from keras.utils import to_categorical
|
|
from sklearn.model_selection import train_test_split
|
|
|
|
# representation of labels
|
|
labels = to_categorical(labels)
|
|
|
|
# split into train and test data
|
|
# one-liner from scikit-learn library
|
|
train_size = 0.8
|
|
test_size = 1 - train_size
|
|
X_train, X_test, Y_train, Y_test = train_test_split(inputs, labels, train_size=train_size,
|
|
test_size=test_size)
|
|
|
|
import tensorflow as tf
|
|
|
|
class ConvolutionalNeuralNetworkTensorflow:
|
|
def __init__(
|
|
self,
|
|
X_train,
|
|
Y_train,
|
|
X_test,
|
|
Y_test,
|
|
n_filters=10,
|
|
n_neurons_connected=50,
|
|
n_categories=10,
|
|
receptive_field=3,
|
|
stride=1,
|
|
padding=1,
|
|
epochs=10,
|
|
batch_size=100,
|
|
eta=0.1,
|
|
lmbd=0.0,
|
|
):
|
|
|
|
self.global_step = tf.Variable(0, dtype=tf.int32, trainable=False, name='global_step')
|
|
|
|
self.X_train = X_train
|
|
self.Y_train = Y_train
|
|
self.X_test = X_test
|
|
self.Y_test = Y_test
|
|
|
|
self.n_inputs, self.input_width, self.input_height, self.depth = X_train.shape
|
|
|
|
self.n_filters = n_filters
|
|
self.n_downsampled = int(self.input_width*self.input_height*n_filters / 4)
|
|
self.n_neurons_connected = n_neurons_connected
|
|
self.n_categories = n_categories
|
|
|
|
self.receptive_field = receptive_field
|
|
self.stride = stride
|
|
self.strides = [stride, stride, stride, stride]
|
|
self.padding = padding
|
|
|
|
self.epochs = epochs
|
|
self.batch_size = batch_size
|
|
self.iterations = self.n_inputs // self.batch_size
|
|
self.eta = eta
|
|
self.lmbd = lmbd
|
|
|
|
self.create_placeholders()
|
|
self.create_CNN()
|
|
self.create_loss()
|
|
self.create_optimiser()
|
|
self.create_accuracy()
|
|
|
|
def create_placeholders(self):
|
|
with tf.name_scope('data'):
|
|
self.X = tf.placeholder(tf.float32, shape=(None, self.input_width, self.input_height, self.depth), name='X_data')
|
|
self.Y = tf.placeholder(tf.float32, shape=(None, self.n_categories), name='Y_data')
|
|
|
|
def create_CNN(self):
|
|
with tf.name_scope('CNN'):
|
|
|
|
# Convolutional layer
|
|
self.W_conv = self.weight_variable([self.receptive_field, self.receptive_field, self.depth, self.n_filters], name='conv', dtype=tf.float32)
|
|
b_conv = self.weight_variable([self.n_filters], name='conv', dtype=tf.float32)
|
|
z_conv = tf.nn.conv2d(self.X, self.W_conv, self.strides, padding='SAME', name='conv') + b_conv
|
|
a_conv = tf.nn.relu(z_conv)
|
|
|
|
# 2x2 max pooling
|
|
a_pool = tf.nn.max_pool(a_conv, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME', name='pool')
|
|
|
|
# Fully connected layer
|
|
a_pool_flat = tf.reshape(a_pool, [-1, self.n_downsampled])
|
|
self.W_fc = self.weight_variable([self.n_downsampled, self.n_neurons_connected], name='fc', dtype=tf.float32)
|
|
b_fc = self.bias_variable([self.n_neurons_connected], name='fc', dtype=tf.float32)
|
|
a_fc = tf.nn.relu(tf.matmul(a_pool_flat, self.W_fc) + b_fc)
|
|
|
|
# Output layer
|
|
self.W_out = self.weight_variable([self.n_neurons_connected, self.n_categories], name='out', dtype=tf.float32)
|
|
b_out = self.bias_variable([self.n_categories], name='out', dtype=tf.float32)
|
|
self.z_out = tf.matmul(a_fc, self.W_out) + b_out
|
|
|
|
def create_loss(self):
|
|
with tf.name_scope('loss'):
|
|
softmax_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=self.Y, logits=self.z_out))
|
|
|
|
regularizer_loss_conv = tf.nn.l2_loss(self.W_conv)
|
|
regularizer_loss_fc = tf.nn.l2_loss(self.W_fc)
|
|
regularizer_loss_out = tf.nn.l2_loss(self.W_out)
|
|
regularizer_loss = self.lmbd*(regularizer_loss_conv + regularizer_loss_fc + regularizer_loss_out)
|
|
|
|
self.loss = softmax_loss + regularizer_loss
|
|
|
|
def create_accuracy(self):
|
|
with tf.name_scope('accuracy'):
|
|
probabilities = tf.nn.softmax(self.z_out)
|
|
predictions = tf.argmax(probabilities, 1)
|
|
labels = tf.argmax(self.Y, 1)
|
|
|
|
correct_predictions = tf.equal(predictions, labels)
|
|
correct_predictions = tf.cast(correct_predictions, tf.float32)
|
|
self.accuracy = tf.reduce_mean(correct_predictions)
|
|
|
|
def create_optimiser(self):
|
|
with tf.name_scope('optimizer'):
|
|
self.optimizer = tf.train.GradientDescentOptimizer(learning_rate=self.eta).minimize(self.loss, global_step=self.global_step)
|
|
|
|
def weight_variable(self, shape, name='', dtype=tf.float32):
|
|
initial = tf.truncated_normal(shape, stddev=0.1)
|
|
return tf.Variable(initial, name=name, dtype=dtype)
|
|
|
|
def bias_variable(self, shape, name='', dtype=tf.float32):
|
|
initial = tf.constant(0.1, shape=shape)
|
|
return tf.Variable(initial, name=name, dtype=dtype)
|
|
|
|
def fit(self):
|
|
data_indices = np.arange(self.n_inputs)
|
|
|
|
with tf.Session() as sess:
|
|
sess.run(tf.global_variables_initializer())
|
|
for i in range(self.epochs):
|
|
for j in range(self.iterations):
|
|
chosen_datapoints = np.random.choice(data_indices, size=self.batch_size, replace=False)
|
|
batch_X, batch_Y = self.X_train[chosen_datapoints], self.Y_train[chosen_datapoints]
|
|
|
|
sess.run([CNN.loss, CNN.optimizer],
|
|
feed_dict={CNN.X: batch_X,
|
|
CNN.Y: batch_Y})
|
|
accuracy = sess.run(CNN.accuracy,
|
|
feed_dict={CNN.X: batch_X,
|
|
CNN.Y: batch_Y})
|
|
step = sess.run(CNN.global_step)
|
|
|
|
self.train_loss, self.train_accuracy = sess.run([CNN.loss, CNN.accuracy],
|
|
feed_dict={CNN.X: self.X_train,
|
|
CNN.Y: self.Y_train})
|
|
|
|
self.test_loss, self.test_accuracy = sess.run([CNN.loss, CNN.accuracy],
|
|
feed_dict={CNN.X: self.X_test,
|
|
CNN.Y: self.Y_test})
|
|
|
|
epochs = 100
|
|
batch_size = 100
|
|
n_filters = 10
|
|
n_neurons_connected = 50
|
|
n_categories = 10
|
|
|
|
eta_vals = np.logspace(-5, 1, 7)
|
|
lmbd_vals = np.logspace(-5, 1, 7)
|
|
CNN_tf = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
|
|
|
|
for i, eta in enumerate(eta_vals):
|
|
for j, lmbd in enumerate(lmbd_vals):
|
|
CNN = ConvolutionalNeuralNetworkTensorflow(X_train, Y_train, X_test, Y_test,
|
|
n_filters=n_filters, n_neurons_connected=n_neurons_connected,
|
|
n_categories=n_categories, epochs=epochs, batch_size=batch_size,
|
|
eta=eta, lmbd=lmbd)
|
|
CNN.fit()
|
|
|
|
print("Learning rate = ", eta)
|
|
print("Lambda = ", lmbd)
|
|
print("Test accuracy: %.3f" % CNN.test_accuracy)
|
|
print()
|
|
|
|
CNN_tf[i][j] = CNN
|
|
|
|
# visual representation of grid search
|
|
# uses seaborn heatmap, could probably do this in matplotlib
|
|
import seaborn as sns
|
|
|
|
sns.set()
|
|
|
|
train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
|
|
test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
|
|
|
|
for i in range(len(eta_vals)):
|
|
for j in range(len(lmbd_vals)):
|
|
CNN = CNN_tf[i][j]
|
|
|
|
train_accuracy[i][j] = CNN.train_accuracy
|
|
test_accuracy[i][j] = CNN.test_accuracy
|
|
|
|
|
|
fig, ax = plt.subplots(figsize = (10, 10))
|
|
sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis")
|
|
ax.set_title("Training Accuracy")
|
|
ax.set_ylabel("$\eta$")
|
|
ax.set_xlabel("$\lambda$")
|
|
plt.show()
|
|
|
|
fig, ax = plt.subplots(figsize = (10, 10))
|
|
sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
|
|
ax.set_title("Test Accuracy")
|
|
ax.set_ylabel("$\eta$")
|
|
ax.set_xlabel("$\lambda$")
|
|
plt.show()
|
|
|
|
from keras.models import Sequential
|
|
from keras.layers.convolutional import Conv2D
|
|
from keras.layers.convolutional import MaxPooling2D
|
|
from keras.layers import Flatten
|
|
from keras.layers import Dense
|
|
from keras.regularizers import l2
|
|
from keras.optimizers import SGD
|
|
|
|
def create_convolutional_neural_network_keras(input_shape, receptive_field,
|
|
n_filters, n_neurons_connected, n_categories,
|
|
eta, lmbd):
|
|
model = Sequential()
|
|
model.add(Conv2D(n_filters, (receptive_field, receptive_field), input_shape=input_shape, padding='same',
|
|
activation='relu', kernel_regularizer=l2(lmbd)))
|
|
model.add(MaxPooling2D(pool_size=(2, 2)))
|
|
model.add(Flatten())
|
|
model.add(Dense(n_neurons_connected, activation='relu', kernel_regularizer=l2(lmbd)))
|
|
model.add(Dense(n_categories, activation='softmax', kernel_regularizer=l2(lmbd)))
|
|
|
|
sgd = SGD(lr=eta)
|
|
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
|
|
|
|
return model
|
|
|
|
epochs = 100
|
|
batch_size = 100
|
|
input_shape = X_train.shape[1:4]
|
|
receptive_field = 3
|
|
n_filters = 10
|
|
n_neurons_connected = 50
|
|
n_categories = 10
|
|
|
|
eta_vals = np.logspace(-5, 1, 7)
|
|
lmbd_vals = np.logspace(-5, 1, 7)
|
|
|
|
CNN_keras = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
|
|
|
|
for i, eta in enumerate(eta_vals):
|
|
for j, lmbd in enumerate(lmbd_vals):
|
|
CNN = create_convolutional_neural_network_keras(input_shape, receptive_field,
|
|
n_filters, n_neurons_connected, n_categories,
|
|
eta, lmbd)
|
|
CNN.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size, verbose=0)
|
|
scores = CNN.evaluate(X_test, Y_test)
|
|
|
|
CNN_keras[i][j] = CNN
|
|
|
|
print("Learning rate = ", eta)
|
|
print("Lambda = ", lmbd)
|
|
print("Test accuracy: %.3f" % scores[1])
|
|
print()
|
|
|
|
# visual representation of grid search
|
|
# uses seaborn heatmap, could probably do this in matplotlib
|
|
import seaborn as sns
|
|
|
|
sns.set()
|
|
|
|
train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
|
|
test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
|
|
|
|
for i in range(len(eta_vals)):
|
|
for j in range(len(lmbd_vals)):
|
|
CNN = CNN_keras[i][j]
|
|
|
|
train_accuracy[i][j] = CNN.evaluate(X_train, Y_train)[1]
|
|
test_accuracy[i][j] = CNN.evaluate(X_test, Y_test)[1]
|
|
|
|
|
|
fig, ax = plt.subplots(figsize = (10, 10))
|
|
sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis")
|
|
ax.set_title("Training Accuracy")
|
|
ax.set_ylabel("$\eta$")
|
|
ax.set_xlabel("$\lambda$")
|
|
plt.show()
|
|
|
|
fig, ax = plt.subplots(figsize = (10, 10))
|
|
sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
|
|
ax.set_title("Test Accuracy")
|
|
ax.set_ylabel("$\eta$")
|
|
ax.set_xlabel("$\lambda$")
|
|
plt.show()
|