lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updating week41

Browse Source
This commit is contained in:
mhjensen
2020-10-09 06:04:27 +02:00
parent 0ffba66be5
commit b09ab9c83c
68 changed files with 8896 additions and 5932 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/src/week41/week41.do.txt
+385 -239
View File
@@ -1100,9 +1100,35 @@ To install tensorflow on Unix/Linux systems, use pip as
pip3 install tensorflow
!ec
and/or if you use _anaconda_, just write (or install from the graphical user interface)
!bc pycod
conda install tensorflow
(current release of CPU-only TensorFlow)
!bc pycod
conda create -n tf tensorflow
conda activate tf
!ec
To install the current release of GPU TensorFlow
!bc pycod
conda create -n tf-gpu tensorflow-gpu
conda activate tf-gpu
!ec
!split
===== Using Keras =====
Keras is a high level "neural network":"https://en.wikipedia.org/wiki/Application_programming_interface"
that supports Tensorflow, CTNK and Theano as backends.
If you have Tensorflow installed Keras is available through the *tf.keras* module.
If you have Anaconda installed you may run the following command
!bc pycod
conda install keras
!ec
Alternatively, if you have Tensorflow or one of the other supported backends install you may use the pip package manager:
!bc pycod
pip install keras
!ec
or look up the "instructions here":"https://keras.io/".
!split
===== Collect and pre-process data =====
@@ -1111,6 +1137,7 @@ conda install tensorflow
# import necessary packages
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from sklearn import datasets
@@ -1153,7 +1180,13 @@ plt.show()
!ec
!bc pycod
from keras.utils import to_categorical
from tensorflow.keras.layers import Input
from tensorflow.keras.models import Sequential #This allows appending layers to existing models
from tensorflow.keras.layers import Dense #This allows defining the characteristics of a particular layer
from tensorflow.keras import optimizers #This allows using whichever optimiser we want (sgd,adam,RMSprop)
from tensorflow.keras import regularizers #This allows using whichever regularizer we want (l1,l2,l1_l2)
from tensorflow.keras.utils import to_categorical #This allows using categorical cross entropy as the cost function
from sklearn.model_selection import train_test_split
# one-hot representation of labels
@@ -1166,245 +1199,9 @@ X_train, X_test, Y_train, Y_test = train_test_split(inputs, labels, train_size=t
test_size=test_size)
!ec
!split
===== Using TensorFlow backend =====
o Define model and architecture
o Choose cost function and optimizer
!bc pycod
import tensorflow as tf
class NeuralNetworkTensorflow:
def __init__(
self,
X_train,
Y_train,
X_test,
Y_test,
n_neurons_layer1=100,
n_neurons_layer2=50,
n_categories=2,
epochs=10,
batch_size=100,
eta=0.1,
lmbd=0.0):
# keep track of number of steps
self.global_step = tf.Variable(0, dtype=tf.int32, trainable=False, name='global_step')
self.X_train = X_train
self.Y_train = Y_train
self.X_test = X_test
self.Y_test = Y_test
self.n_inputs = X_train.shape[0]
self.n_features = X_train.shape[1]
self.n_neurons_layer1 = n_neurons_layer1
self.n_neurons_layer2 = n_neurons_layer2
self.n_categories = n_categories
self.epochs = epochs
self.batch_size = batch_size
self.iterations = self.n_inputs // self.batch_size
self.eta = eta
self.lmbd = lmbd
# build network piece by piece
# name scopes (with) are used to enforce creation of new variables
# https://www.tensorflow.org/guide/variables
self.create_placeholders()
self.create_DNN()
self.create_loss()
self.create_optimiser()
self.create_accuracy()
def create_placeholders(self):
# placeholders are fine here, but "Datasets" are the preferred method
# of streaming data into a model
with tf.name_scope('data'):
self.X = tf.placeholder(tf.float32, shape=(None, self.n_features), name='X_data')
self.Y = tf.placeholder(tf.float32, shape=(None, self.n_categories), name='Y_data')
def create_DNN(self):
with tf.name_scope('DNN'):
# the weights are stored to calculate regularization loss later
# Fully connected layer 1
self.W_fc1 = self.weight_variable([self.n_features, self.n_neurons_layer1], name='fc1', dtype=tf.float32)
b_fc1 = self.bias_variable([self.n_neurons_layer1], name='fc1', dtype=tf.float32)
a_fc1 = tf.nn.sigmoid(tf.matmul(self.X, self.W_fc1) + b_fc1)
# Fully connected layer 2
self.W_fc2 = self.weight_variable([self.n_neurons_layer1, self.n_neurons_layer2], name='fc2', dtype=tf.float32)
b_fc2 = self.bias_variable([self.n_neurons_layer2], name='fc2', dtype=tf.float32)
a_fc2 = tf.nn.sigmoid(tf.matmul(a_fc1, self.W_fc2) + b_fc2)
# Output layer
self.W_out = self.weight_variable([self.n_neurons_layer2, self.n_categories], name='out', dtype=tf.float32)
b_out = self.bias_variable([self.n_categories], name='out', dtype=tf.float32)
self.z_out = tf.matmul(a_fc2, self.W_out) + b_out
def create_loss(self):
with tf.name_scope('loss'):
softmax_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=self.Y, logits=self.z_out))
regularizer_loss_fc1 = tf.nn.l2_loss(self.W_fc1)
regularizer_loss_fc2 = tf.nn.l2_loss(self.W_fc2)
regularizer_loss_out = tf.nn.l2_loss(self.W_out)
regularizer_loss = self.lmbd*(regularizer_loss_fc1 + regularizer_loss_fc2 + regularizer_loss_out)
self.loss = softmax_loss + regularizer_loss
def create_accuracy(self):
with tf.name_scope('accuracy'):
probabilities = tf.nn.softmax(self.z_out)
predictions = tf.argmax(probabilities, axis=1)
labels = tf.argmax(self.Y, axis=1)
correct_predictions = tf.equal(predictions, labels)
correct_predictions = tf.cast(correct_predictions, tf.float32)
self.accuracy = tf.reduce_mean(correct_predictions)
def create_optimiser(self):
with tf.name_scope('optimizer'):
self.optimizer = tf.train.GradientDescentOptimizer(learning_rate=self.eta).minimize(self.loss, global_step=self.global_step)
def weight_variable(self, shape, name='', dtype=tf.float32):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial, name=name, dtype=dtype)
def bias_variable(self, shape, name='', dtype=tf.float32):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial, name=name, dtype=dtype)
def fit(self):
data_indices = np.arange(self.n_inputs)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(self.epochs):
for j in range(self.iterations):
chosen_datapoints = np.random.choice(data_indices, size=self.batch_size, replace=False)
batch_X, batch_Y = self.X_train[chosen_datapoints], self.Y_train[chosen_datapoints]
sess.run([DNN.loss, DNN.optimizer],
feed_dict={DNN.X: batch_X,
DNN.Y: batch_Y})
accuracy = sess.run(DNN.accuracy,
feed_dict={DNN.X: batch_X,
DNN.Y: batch_Y})
step = sess.run(DNN.global_step)
self.train_loss, self.train_accuracy = sess.run([DNN.loss, DNN.accuracy],
feed_dict={DNN.X: self.X_train,
DNN.Y: self.Y_train})
self.test_loss, self.test_accuracy = sess.run([DNN.loss, DNN.accuracy],
feed_dict={DNN.X: self.X_test,
DNN.Y: self.Y_test})
!ec
!split
===== Optimizing and using gradient descent =====
!bc pycod
epochs = 100
batch_size = 100
n_neurons_layer1 = 100
n_neurons_layer2 = 50
n_categories = 10
eta_vals = np.logspace(-5, 1, 7)
lmbd_vals = np.logspace(-5, 1, 7)
!ec
!bc pycod
DNN_tf = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
for i, eta in enumerate(eta_vals):
for j, lmbd in enumerate(lmbd_vals):
DNN = NeuralNetworkTensorflow(X_train, Y_train, X_test, Y_test,
n_neurons_layer1, n_neurons_layer2, n_categories,
epochs=epochs, batch_size=batch_size, eta=eta, lmbd=lmbd)
DNN.fit()
DNN_tf[i][j] = DNN
print("Learning rate = ", eta)
print("Lambda = ", lmbd)
print("Test accuracy: %.3f" % DNN.test_accuracy)
print()
!ec
!bc pycod
# optional
# visual representation of grid search
# uses seaborn heatmap, could probably do this in matplotlib
import seaborn as sns
sns.set()
train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
for i in range(len(eta_vals)):
for j in range(len(lmbd_vals)):
DNN = DNN_tf[i][j]
train_accuracy[i][j] = DNN.train_accuracy
test_accuracy[i][j] = DNN.test_accuracy
fig, ax = plt.subplots(figsize = (10, 10))
sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis")
ax.set_title("Training Accuracy")
ax.set_ylabel("$\eta$")
ax.set_xlabel("$\lambda$")
plt.show()
fig, ax = plt.subplots(figsize = (10, 10))
sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
ax.set_title("Test Accuracy")
ax.set_ylabel("$\eta$")
ax.set_xlabel("$\lambda$")
plt.show()
!ec
!bc pycod
# optional
# we can use log files to visualize our graph in Tensorboard
writer = tf.summary.FileWriter('logs/')
writer.add_graph(tf.get_default_graph())
!ec
!split
===== Using Keras =====
Keras is a high level "neural network":"https://en.wikipedia.org/wiki/Application_programming_interface"
that supports Tensorflow, CTNK and Theano as backends.
If you have Tensorflow installed Keras is available through the *tf.keras* module.
If you have Anaconda installed you may run the following command
!bc pycod
conda install keras
!ec
Alternatively, if you have Tensorflow or one of the other supported backends install you may use the pip package manager:
!bc pycod
pip install keras
!ec
or look up the "instructions here":"https://keras.io/".
!bc pycod
import tensorflow as tf
from tensorflow.keras.layers import Input
from tensorflow.keras.models import Sequential #This allows appending layers to existing models
from tensorflow.keras.layers import Dense #This allows defining the characteristics of a particular layer
from tensorflow.keras import optimizers #This allows using whichever optimiser we want (sgd,adam,RMSprop)
from tensorflow.keras import regularizers #This allows using whichever regularizer we want (l1,l2,l1_l2)
from tensorflow.keras.utils import to_categorical #This allows using categorical cross entropy as the cost function
def create_neural_network_keras(n_neurons_layer1, n_neurons_layer2, n_categories, eta, lmbd):
model = Sequential()
@@ -1648,6 +1445,47 @@ plot_data(eta,n_neuron,Test_accuracy, 'testing')
!ec
!split
===== Fine-tuning neural network hyperparameters =====
The flexibility of neural networks is also one of their main
drawbacks: there are many hyperparameters to tweak. Not only can you
use any imaginable network topology (how neurons/nodes are interconnected),
but even in a simple FFNN you can change the number of layers, the
number of neurons per layer, the type of activation function to use in
each layer, the weight initialization logic, the stochastic gradient optmized and much more. How do you
know what combination of hyperparameters is the best for your task?
* You can use grid search with cross-validation to find the right hyperparameters.
However,since there are many hyperparameters to tune, and since
training a neural network on a large dataset takes a lot of time, you
will only be able to explore a tiny part of the hyperparameter space.
* You can use randomized search.
* Or use tools like "Oscar":"http://oscar.calldesk.ai/", which implements more complex algorithms to help you find a good set of hyperparameters quickly.
!split
===== Hidden layers =====
For many problems you can start with just one or two hidden layers and it will work just fine.
For the MNIST data set you ca easily get a high accuracy using just one hidden layer with a
few hundred neurons.
You can reach for this data set above 98% accuracy using two hidden layers with the same total amount of
neurons, in roughly the same amount of training time.
For more complex problems, you can gradually
ramp up the number of hidden layers, until you start overfitting the training set. Very complex tasks, such
as large image classification or speech recognition, typically require networks with dozens of layers
and they need a huge amount
of training data. However, you will rarely have to train such networks from scratch: it is much more
common to reuse parts of a pretrained state-of-the-art network that performs a similar task.
!split
===== Which activation function should I use? =====
@@ -1674,6 +1512,19 @@ mostly encountered in recurrent neural networks. More generally, deep
neural networks suffer from unstable gradients, different layers may
learn at widely different speeds
!split
===== More on activation functions, output layers =====
In most cases you can use the ReLU activation function in the hidden layers (or one of its variants).
It is a bit faster to compute than other activation functions, and the gradient descent optimization does in general not get stuck.
_For the output layer:_
* For classification the softmax activation function is generally a good choice for classification tasks (when the classes are mutually exclusive).
* For regression tasks, you can simply use no activation function at all.
!split
===== Is the Logistic activation function (Sigmoid) our choice? =====
@@ -1775,6 +1626,43 @@ $\alpha$ of $0.01$ for the leaky ReLU, and $1$ for ELU. If you have
spare time and computing power, you can use cross-validation or
bootstrap to evaluate other activation functions.
!split
===== Batch Normalization =====
Batch Normalization
aims to address the vanishing/exploding gradients problems, and more generally the problem that the
distribution of each layer’s inputs changes during training, as the parameters of the previous layers change.
The technique consists of adding an operation in the model just before the activation function of each
layer, simply zero-centering and normalizing the inputs, then scaling and shifting the result using two new
parameters per layer (one for scaling, the other for shifting). In other words, this operation lets the model
learn the optimal scale and mean of the inputs for each layer.
In order to zero-center and normalize the inputs, the algorithm needs to estimate the inputs’ mean and
standard deviation. It does so by evaluating the mean and standard deviation of the inputs over the current
mini-batch, from this the name batch normalization.
!split
===== Dropout =====
It is a fairly simple algorithm: at every training step, every neuron (including the input neurons but
excluding the output neurons) has a probability $p$ of being temporarily dropped out, meaning it will be
entirely ignored during this training step, but it may be active during the next step.
The
hyperparameter $p$ is called the dropout rate, and it is typically set to 50%. After training, the neurons are not dropped anymore.
It is viewed as one of the most popular regularization techniques.
!split
===== Gradient Clipping =====
A popular technique to lessen the exploding gradients problem is to simply clip the gradients during
backpropagation so that they never exceed some threshold (this is mostly useful for recurrent neural
networks).
This technique is called Gradient Clipping.
In general however, Batch
Normalization is preferred.
!split
===== A top-down perspective on Neural networks =====
@@ -1979,3 +1867,261 @@ the course
and the slides of "CS231":"http://cs231n.github.io/convolutional-networks/" which is taught at Stanford University (consistently ranked as one of the top computer science programs in the world). "Michael Nielsen's book is a must read, in particular chapter 6 which deals with CNNs":"http://neuralnetworksanddeeplearning.com/chap6.html".
!split
===== CNNs in more detail, building convolutional neural networks in Tensorflow and Keras =====
As discussed above, CNNs are neural networks built from the assumption that the inputs
to the network are 2D images. This is important because the number of features or pixels in images
grows very fast with the image size, and an enormous number of weights and biases are needed in order to build an accurate network.
As before, we still have our input, a hidden layer and an output. What's novel about convolutional networks
are the _convolutional_ and _pooling_ layers stacked in pairs between the input and the hidden layer.
In addition, the data is no longer represented as a 2D feature matrix, instead each input is a number of 2D
matrices, typically 1 for each color dimension (Red, Green, Blue).
!split
===== Setting it up =====
It means that to represent the entire
dataset of images, we require a 4D matrix or _tensor_. This tensor has the dimensions:
!bt
\[
(n_{inputs},\, n_{pixels, width},\, n_{pixels, height},\, depth) .
\]
!et
!split
===== The MNIST dataset again =====
The MNIST dataset consists of grayscale images with a pixel size of
$28\times 28$, meaning we require $28 \times 28 = 724$ weights to each
neuron in the first hidden layer.
If we were to analyze images of size $128\times 128$ we would require
$128 \times 128 = 16384$ weights to each neuron. Even worse if we were
dealing with color images, as most images are, we have an image matrix
of size $128\times 128$ for each color dimension (Red, Green, Blue),
meaning 3 times the number of weights $= 49152$ are required for every
single neuron in the first hidden layer.
!split
===== Strong correlations =====
Images typically have strong local correlations, meaning that a small
part of the image varies little from its neighboring regions. If for
example we have an image of a blue car, we can roughly assume that a
small blue part of the image is surrounded by other blue regions.
Therefore, instead of connecting every single pixel to a neuron in the
first hidden layer, as we have previously done with deep neural
networks, we can instead connect each neuron to a small part of the
image (in all 3 RGB depth dimensions). The size of each small area is
fixed, and known as a "receptive":"https://en.wikipedia.org/wiki/Receptive_field".
!split
===== Layers of a CNN =====
The layers of a convolutional neural network arrange neurons in 3D: width, height and depth.
The input image is typically a square matrix of depth 3.
A _convolution_ is performed on the image which outputs
a 3D volume of neurons. The weights to the input are arranged in a number of 2D matrices, known as _filters_.
Each filter slides along the input image, taking the dot product
between each small part of the image and the filter, in all depth
dimensions. This is then passed through a non-linear function,
typically the _Rectified Linear (ReLu)_ function, which serves as the
activation of the neurons in the first convolutional layer. This is
further passed through a _pooling layer_, which reduces the size of the
convolutional layer, e.g. by taking the maximum or average across some
small regions, and this serves as input to the next convolutional
layer.
!split
===== Systematic reduction =====
By systematically reducing the size of the input volume, through
convolution and pooling, the network should create representations of
small parts of the input, and then from them assemble representations
of larger areas. The final pooling layer is flattened to serve as
input to a hidden layer, such that each neuron in the final pooling
layer is connected to every single neuron in the hidden layer. This
then serves as input to the output layer, e.g. a softmax output for
classification.
!split
===== Prerequisites: Collect and pre-process data =====
!bc pycod
# import necessary packages
import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets
# ensure the same random numbers appear every time
np.random.seed(0)
# display images in notebook
%matplotlib inline
plt.rcParams['figure.figsize'] = (12,12)
# download MNIST dataset
digits = datasets.load_digits()
# define inputs and labels
inputs = digits.images
labels = digits.target
# RGB images have a depth of 3
# our images are grayscale so they should have a depth of 1
inputs = inputs[:,:,:,np.newaxis]
print("inputs = (n_inputs, pixel_width, pixel_height, depth) = " + str(inputs.shape))
print("labels = (n_inputs) = " + str(labels.shape))
# choose some random images to display
n_inputs = len(inputs)
indices = np.arange(n_inputs)
random_indices = np.random.choice(indices, size=5)
for i, image in enumerate(digits.images[random_indices]):
plt.subplot(1, 5, i+1)
plt.axis('off')
plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest')
plt.title("Label: %d" % digits.target[random_indices[i]])
plt.show()
!ec
!split
===== Importing Keras and Tensorflow =====
!bc pycod
from keras.utils import to_categorical
from sklearn.model_selection import train_test_split
# representation of labels
labels = to_categorical(labels)
# split into train and test data
# one-liner from scikit-learn library
train_size = 0.8
test_size = 1 - train_size
X_train, X_test, Y_train, Y_test = train_test_split(inputs, labels, train_size=train_size,
test_size=test_size)
!ec
!split
===== Running with Keras =====
!bc pycod
from keras.models import Sequential
from keras.layers.convolutional import Conv2D
from keras.layers.convolutional import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense
from keras.regularizers import l2
from keras.optimizers import SGD
def create_convolutional_neural_network_keras(input_shape, receptive_field,
n_filters, n_neurons_connected, n_categories,
eta, lmbd):
model = Sequential()
model.add(Conv2D(n_filters, (receptive_field, receptive_field), input_shape=input_shape, padding='same',
activation='relu', kernel_regularizer=l2(lmbd)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(n_neurons_connected, activation='relu', kernel_regularizer=l2(lmbd)))
model.add(Dense(n_categories, activation='softmax', kernel_regularizer=l2(lmbd)))
sgd = SGD(lr=eta)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
return model
epochs = 100
batch_size = 100
input_shape = X_train.shape[1:4]
receptive_field = 3
n_filters = 10
n_neurons_connected = 50
n_categories = 10
eta_vals = np.logspace(-5, 1, 7)
lmbd_vals = np.logspace(-5, 1, 7)
!ec
!split
===== Final part =====
!bc pycod
CNN_keras = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
for i, eta in enumerate(eta_vals):
for j, lmbd in enumerate(lmbd_vals):
CNN = create_convolutional_neural_network_keras(input_shape, receptive_field,
n_filters, n_neurons_connected, n_categories,
eta, lmbd)
CNN.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size, verbose=0)
scores = CNN.evaluate(X_test, Y_test)
CNN_keras[i][j] = CNN
print("Learning rate = ", eta)
print("Lambda = ", lmbd)
print("Test accuracy: %.3f" % scores[1])
print()
!ec
!split
===== Final visualization =====
!bc
# visual representation of grid search
# uses seaborn heatmap, could probably do this in matplotlib
import seaborn as sns
sns.set()
train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
for i in range(len(eta_vals)):
for j in range(len(lmbd_vals)):
CNN = CNN_keras[i][j]
train_accuracy[i][j] = CNN.evaluate(X_train, Y_train)[1]
test_accuracy[i][j] = CNN.evaluate(X_test, Y_test)[1]
fig, ax = plt.subplots(figsize = (10, 10))
sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis")
ax.set_title("Training Accuracy")
ax.set_ylabel("$\eta$")
ax.set_xlabel("$\lambda$")
plt.show()
fig, ax = plt.subplots(figsize = (10, 10))
sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
ax.set_title("Test Accuracy")
ax.set_ylabel("$\eta$")
ax.set_xlabel("$\lambda$")
plt.show()
!ec
!split
===== Fun links =====
o "Self-Driving cars using a convolutional neural network":"https://arxiv.org/abs/1604.07316"
o "Abstract art using convolutional neural networks":"https://deepdreamgenerator.com/"