diff --git a/doc/Programs/ANN/cnn.py b/doc/Programs/ANN/cnn.py new file mode 100644 index 000000000..f81b35353 --- /dev/null +++ b/doc/Programs/ANN/cnn.py @@ -0,0 +1,330 @@ +# import necessary packages +import numpy as np +import matplotlib.pyplot as plt +from sklearn import datasets + + +# ensure the same random numbers appear every time +np.random.seed(0) + +# display images in notebook +plt.rcParams['figure.figsize'] = (12,12) + + +# download MNIST dataset +digits = datasets.load_digits() + +# define inputs and labels +inputs = digits.images +labels = digits.target + +# RGB images have a depth of 3 +# our images are grayscale so they should have a depth of 1 +inputs = inputs[:,:,:,np.newaxis] + +print("inputs = (n_inputs, pixel_width, pixel_height, depth) = " + str(inputs.shape)) +print("labels = (n_inputs) = " + str(labels.shape)) + + +# choose some random images to display +n_inputs = len(inputs) +indices = np.arange(n_inputs) +random_indices = np.random.choice(indices, size=5) + +for i, image in enumerate(digits.images[random_indices]): + plt.subplot(1, 5, i+1) + plt.axis('off') + plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest') + plt.title("Label: %d" % digits.target[random_indices[i]]) +plt.show() + +from keras.utils import to_categorical +from sklearn.model_selection import train_test_split + +# representation of labels +labels = to_categorical(labels) + +# split into train and test data +# one-liner from scikit-learn library +train_size = 0.8 +test_size = 1 - train_size +X_train, X_test, Y_train, Y_test = train_test_split(inputs, labels, train_size=train_size, + test_size=test_size) + +import tensorflow as tf + +class ConvolutionalNeuralNetworkTensorflow: + def __init__( + self, + X_train, + Y_train, + X_test, + Y_test, + n_filters=10, + n_neurons_connected=50, + n_categories=10, + receptive_field=3, + stride=1, + padding=1, + epochs=10, + batch_size=100, + eta=0.1, + lmbd=0.0, + ): + + self.global_step = tf.Variable(0, dtype=tf.int32, trainable=False, name='global_step') + + self.X_train = X_train + self.Y_train = Y_train + self.X_test = X_test + self.Y_test = Y_test + + self.n_inputs, self.input_width, self.input_height, self.depth = X_train.shape + + self.n_filters = n_filters + self.n_downsampled = int(self.input_width*self.input_height*n_filters / 4) + self.n_neurons_connected = n_neurons_connected + self.n_categories = n_categories + + self.receptive_field = receptive_field + self.stride = stride + self.strides = [stride, stride, stride, stride] + self.padding = padding + + self.epochs = epochs + self.batch_size = batch_size + self.iterations = self.n_inputs // self.batch_size + self.eta = eta + self.lmbd = lmbd + + self.create_placeholders() + self.create_CNN() + self.create_loss() + self.create_optimiser() + self.create_accuracy() + + def create_placeholders(self): + with tf.name_scope('data'): + self.X = tf.placeholder(tf.float32, shape=(None, self.input_width, self.input_height, self.depth), name='X_data') + self.Y = tf.placeholder(tf.float32, shape=(None, self.n_categories), name='Y_data') + + def create_CNN(self): + with tf.name_scope('CNN'): + + # Convolutional layer + self.W_conv = self.weight_variable([self.receptive_field, self.receptive_field, self.depth, self.n_filters], name='conv', dtype=tf.float32) + b_conv = self.weight_variable([self.n_filters], name='conv', dtype=tf.float32) + z_conv = tf.nn.conv2d(self.X, self.W_conv, self.strides, padding='SAME', name='conv') + b_conv + a_conv = tf.nn.relu(z_conv) + + # 2x2 max pooling + a_pool = tf.nn.max_pool(a_conv, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME', name='pool') + + # Fully connected layer + a_pool_flat = tf.reshape(a_pool, [-1, self.n_downsampled]) + self.W_fc = self.weight_variable([self.n_downsampled, self.n_neurons_connected], name='fc', dtype=tf.float32) + b_fc = self.bias_variable([self.n_neurons_connected], name='fc', dtype=tf.float32) + a_fc = tf.nn.relu(tf.matmul(a_pool_flat, self.W_fc) + b_fc) + + # Output layer + self.W_out = self.weight_variable([self.n_neurons_connected, self.n_categories], name='out', dtype=tf.float32) + b_out = self.bias_variable([self.n_categories], name='out', dtype=tf.float32) + self.z_out = tf.matmul(a_fc, self.W_out) + b_out + + def create_loss(self): + with tf.name_scope('loss'): + softmax_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=self.Y, logits=self.z_out)) + + regularizer_loss_conv = tf.nn.l2_loss(self.W_conv) + regularizer_loss_fc = tf.nn.l2_loss(self.W_fc) + regularizer_loss_out = tf.nn.l2_loss(self.W_out) + regularizer_loss = self.lmbd*(regularizer_loss_conv + regularizer_loss_fc + regularizer_loss_out) + + self.loss = softmax_loss + regularizer_loss + + def create_accuracy(self): + with tf.name_scope('accuracy'): + probabilities = tf.nn.softmax(self.z_out) + predictions = tf.argmax(probabilities, 1) + labels = tf.argmax(self.Y, 1) + + correct_predictions = tf.equal(predictions, labels) + correct_predictions = tf.cast(correct_predictions, tf.float32) + self.accuracy = tf.reduce_mean(correct_predictions) + + def create_optimiser(self): + with tf.name_scope('optimizer'): + self.optimizer = tf.train.GradientDescentOptimizer(learning_rate=self.eta).minimize(self.loss, global_step=self.global_step) + + def weight_variable(self, shape, name='', dtype=tf.float32): + initial = tf.truncated_normal(shape, stddev=0.1) + return tf.Variable(initial, name=name, dtype=dtype) + + def bias_variable(self, shape, name='', dtype=tf.float32): + initial = tf.constant(0.1, shape=shape) + return tf.Variable(initial, name=name, dtype=dtype) + + def fit(self): + data_indices = np.arange(self.n_inputs) + + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + for i in range(self.epochs): + for j in range(self.iterations): + chosen_datapoints = np.random.choice(data_indices, size=self.batch_size, replace=False) + batch_X, batch_Y = self.X_train[chosen_datapoints], self.Y_train[chosen_datapoints] + + sess.run([CNN.loss, CNN.optimizer], + feed_dict={CNN.X: batch_X, + CNN.Y: batch_Y}) + accuracy = sess.run(CNN.accuracy, + feed_dict={CNN.X: batch_X, + CNN.Y: batch_Y}) + step = sess.run(CNN.global_step) + + self.train_loss, self.train_accuracy = sess.run([CNN.loss, CNN.accuracy], + feed_dict={CNN.X: self.X_train, + CNN.Y: self.Y_train}) + + self.test_loss, self.test_accuracy = sess.run([CNN.loss, CNN.accuracy], + feed_dict={CNN.X: self.X_test, + CNN.Y: self.Y_test}) + +epochs = 100 +batch_size = 100 +n_filters = 10 +n_neurons_connected = 50 +n_categories = 10 + +eta_vals = np.logspace(-5, 1, 7) +lmbd_vals = np.logspace(-5, 1, 7) +CNN_tf = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object) + +for i, eta in enumerate(eta_vals): + for j, lmbd in enumerate(lmbd_vals): + CNN = ConvolutionalNeuralNetworkTensorflow(X_train, Y_train, X_test, Y_test, + n_filters=n_filters, n_neurons_connected=n_neurons_connected, + n_categories=n_categories, epochs=epochs, batch_size=batch_size, + eta=eta, lmbd=lmbd) + CNN.fit() + + print("Learning rate = ", eta) + print("Lambda = ", lmbd) + print("Test accuracy: %.3f" % CNN.test_accuracy) + print() + + CNN_tf[i][j] = CNN + +# visual representation of grid search +# uses seaborn heatmap, could probably do this in matplotlib +import seaborn as sns + +sns.set() + +train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) +test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) + +for i in range(len(eta_vals)): + for j in range(len(lmbd_vals)): + CNN = CNN_tf[i][j] + + train_accuracy[i][j] = CNN.train_accuracy + test_accuracy[i][j] = CNN.test_accuracy + + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Training Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Test Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() + +from keras.models import Sequential +from keras.layers.convolutional import Conv2D +from keras.layers.convolutional import MaxPooling2D +from keras.layers import Flatten +from keras.layers import Dense +from keras.regularizers import l2 +from keras.optimizers import SGD + +def create_convolutional_neural_network_keras(input_shape, receptive_field, + n_filters, n_neurons_connected, n_categories, + eta, lmbd): + model = Sequential() + model.add(Conv2D(n_filters, (receptive_field, receptive_field), input_shape=input_shape, padding='same', + activation='relu', kernel_regularizer=l2(lmbd))) + model.add(MaxPooling2D(pool_size=(2, 2))) + model.add(Flatten()) + model.add(Dense(n_neurons_connected, activation='relu', kernel_regularizer=l2(lmbd))) + model.add(Dense(n_categories, activation='softmax', kernel_regularizer=l2(lmbd))) + + sgd = SGD(lr=eta) + model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) + + return model + +epochs = 100 +batch_size = 100 +input_shape = X_train.shape[1:4] +receptive_field = 3 +n_filters = 10 +n_neurons_connected = 50 +n_categories = 10 + +eta_vals = np.logspace(-5, 1, 7) +lmbd_vals = np.logspace(-5, 1, 7) + +CNN_keras = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object) + +for i, eta in enumerate(eta_vals): + for j, lmbd in enumerate(lmbd_vals): + CNN = create_convolutional_neural_network_keras(input_shape, receptive_field, + n_filters, n_neurons_connected, n_categories, + eta, lmbd) + CNN.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size, verbose=0) + scores = CNN.evaluate(X_test, Y_test) + + CNN_keras[i][j] = CNN + + print("Learning rate = ", eta) + print("Lambda = ", lmbd) + print("Test accuracy: %.3f" % scores[1]) + print() + +# visual representation of grid search +# uses seaborn heatmap, could probably do this in matplotlib +import seaborn as sns + +sns.set() + +train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) +test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) + +for i in range(len(eta_vals)): + for j in range(len(lmbd_vals)): + CNN = CNN_keras[i][j] + + train_accuracy[i][j] = CNN.evaluate(X_train, Y_train)[1] + test_accuracy[i][j] = CNN.evaluate(X_test, Y_test)[1] + + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Training Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Test Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() diff --git a/doc/Programs/ANN/cnnkeras.py b/doc/Programs/ANN/cnnkeras.py new file mode 100644 index 000000000..bf4dd5c95 --- /dev/null +++ b/doc/Programs/ANN/cnnkeras.py @@ -0,0 +1,138 @@ +# import necessary packages +import numpy as np +import matplotlib.pyplot as plt +from sklearn import datasets + + +# ensure the same random numbers appear every time +np.random.seed(0) + +# display images in notebook +plt.rcParams['figure.figsize'] = (12,12) + + +# download MNIST dataset +digits = datasets.load_digits() + +# define inputs and labels +inputs = digits.images +labels = digits.target + +# RGB images have a depth of 3 +# our images are grayscale so they should have a depth of 1 +inputs = inputs[:,:,:,np.newaxis] + +print("inputs = (n_inputs, pixel_width, pixel_height, depth) = " + str(inputs.shape)) +print("labels = (n_inputs) = " + str(labels.shape)) + + +# choose some random images to display +n_inputs = len(inputs) +indices = np.arange(n_inputs) +random_indices = np.random.choice(indices, size=5) + +for i, image in enumerate(digits.images[random_indices]): + plt.subplot(1, 5, i+1) + plt.axis('off') + plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest') + plt.title("Label: %d" % digits.target[random_indices[i]]) +plt.show() + +from keras.utils import to_categorical +from sklearn.model_selection import train_test_split + +# representation of labels +labels = to_categorical(labels) + +# split into train and test data +# one-liner from scikit-learn library +train_size = 0.8 +test_size = 1 - train_size +X_train, X_test, Y_train, Y_test = train_test_split(inputs, labels, train_size=train_size, + test_size=test_size) + +import tensorflow as tf + + +from keras.models import Sequential +from keras.layers.convolutional import Conv2D +from keras.layers.convolutional import MaxPooling2D +from keras.layers import Flatten +from keras.layers import Dense +from keras.regularizers import l2 +from keras.optimizers import SGD + +def create_convolutional_neural_network_keras(input_shape, receptive_field, + n_filters, n_neurons_connected, n_categories, + eta, lmbd): + model = Sequential() + model.add(Conv2D(n_filters, (receptive_field, receptive_field), input_shape=input_shape, padding='same', + activation='relu', kernel_regularizer=l2(lmbd))) + model.add(MaxPooling2D(pool_size=(2, 2))) + model.add(Flatten()) + model.add(Dense(n_neurons_connected, activation='relu', kernel_regularizer=l2(lmbd))) + model.add(Dense(n_categories, activation='softmax', kernel_regularizer=l2(lmbd))) + + sgd = SGD(lr=eta) + model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) + + return model + +epochs = 100 +batch_size = 100 +input_shape = X_train.shape[1:4] +receptive_field = 3 +n_filters = 10 +n_neurons_connected = 50 +n_categories = 10 + +eta_vals = np.logspace(-5, 1, 7) +lmbd_vals = np.logspace(-5, 1, 7) + +CNN_keras = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object) + +for i, eta in enumerate(eta_vals): + for j, lmbd in enumerate(lmbd_vals): + CNN = create_convolutional_neural_network_keras(input_shape, receptive_field, + n_filters, n_neurons_connected, n_categories, + eta, lmbd) + CNN.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size, verbose=0) + scores = CNN.evaluate(X_test, Y_test) + + CNN_keras[i][j] = CNN + + print("Learning rate = ", eta) + print("Lambda = ", lmbd) + print("Test accuracy: %.3f" % scores[1]) + print() + +# visual representation of grid search +# uses seaborn heatmap, could probably do this in matplotlib +import seaborn as sns + +sns.set() + +train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) +test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) + +for i in range(len(eta_vals)): + for j in range(len(lmbd_vals)): + CNN = CNN_keras[i][j] + + train_accuracy[i][j] = CNN.evaluate(X_train, Y_train)[1] + test_accuracy[i][j] = CNN.evaluate(X_test, Y_test)[1] + + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Training Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Test Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() diff --git a/doc/pub/NeuralNet/ipynb/NeuralNet.ipynb b/doc/pub/NeuralNet/ipynb/NeuralNet.ipynb index 097b9083c..0c0f659fd 100644 --- a/doc/pub/NeuralNet/ipynb/NeuralNet.ipynb +++ b/doc/pub/NeuralNet/ipynb/NeuralNet.ipynb @@ -597,9 +597,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", @@ -1691,9 +1689,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# import necessary packages\n", @@ -1760,9 +1756,7 @@ { "cell_type": "code", "execution_count": 3, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.model_selection import train_test_split\n", @@ -1884,9 +1878,7 @@ { "cell_type": "code", "execution_count": 4, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# building our neural network\n", @@ -1963,9 +1955,7 @@ { "cell_type": "code", "execution_count": 5, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# setup the feed-forward pass, subscript h = hidden layer\n", @@ -2130,9 +2120,7 @@ { "cell_type": "code", "execution_count": 6, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# to categorical turns our integer vector into a onehot representation\n", @@ -2231,9 +2219,7 @@ { "cell_type": "code", "execution_count": 7, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "class NeuralNetwork:\n", @@ -2356,9 +2342,7 @@ { "cell_type": "code", "execution_count": 8, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "epochs = 100\n", @@ -2392,9 +2376,7 @@ { "cell_type": "code", "execution_count": 9, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "eta_vals = np.logspace(-5, 1, 7)\n", @@ -2429,9 +2411,7 @@ { "cell_type": "code", "execution_count": 10, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# visual representation of grid search\n", @@ -2491,9 +2471,7 @@ { "cell_type": "code", "execution_count": 11, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.neural_network import MLPClassifier\n", @@ -2524,9 +2502,7 @@ { "cell_type": "code", "execution_count": 12, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# optional\n", @@ -2609,9 +2585,7 @@ { "cell_type": "code", "execution_count": 13, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "pip3 install tensorflow" @@ -2627,9 +2601,7 @@ { "cell_type": "code", "execution_count": 14, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "conda install tensorflow" @@ -2645,9 +2617,7 @@ { "cell_type": "code", "execution_count": 15, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# import necessary packages\n", @@ -2697,9 +2667,7 @@ { "cell_type": "code", "execution_count": 16, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "from keras.utils import to_categorical\n", @@ -2729,9 +2697,7 @@ { "cell_type": "code", "execution_count": 17, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "import tensorflow as tf\n", @@ -2877,9 +2843,7 @@ { "cell_type": "code", "execution_count": 18, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "epochs = 100\n", @@ -2894,9 +2858,7 @@ { "cell_type": "code", "execution_count": 19, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "DNN_tf = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)\n", @@ -2919,9 +2881,7 @@ { "cell_type": "code", "execution_count": 20, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# optional\n", @@ -2960,9 +2920,7 @@ { "cell_type": "code", "execution_count": 21, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# optional\n", @@ -2986,9 +2944,7 @@ { "cell_type": "code", "execution_count": 22, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "conda install keras" @@ -3004,9 +2960,7 @@ { "cell_type": "code", "execution_count": 23, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "pip3 install keras" @@ -3022,9 +2976,7 @@ { "cell_type": "code", "execution_count": 24, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "from keras.models import Sequential\n", @@ -3047,9 +2999,7 @@ { "cell_type": "code", "execution_count": 25, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "DNN_keras = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)\n", @@ -3072,9 +3022,7 @@ { "cell_type": "code", "execution_count": 26, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# optional\n", @@ -3549,11 +3497,28 @@ }, { "cell_type": "code", - "execution_count": 27, - "metadata": { - "collapsed": false - }, - "outputs": [], + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "inputs = (n_inputs, pixel_width, pixel_height, depth) = (1797, 8, 8, 1)\n", + "labels = (n_inputs) = (1797,)\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAArkAAACiCAYAAABF0NXFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAC3NJREFUeJzt3V9opflZB/DnaWeh1j9zXBQR25loRWG92NyIipU5A4IgSALLimDbyYiCV2YW9UbQZKReiBebES/0ajNdwYoWEtBFEJ1EtlW0sBnwZkFK2rVYbGVPuq0iWn9enAyEZXY3+7znzDnzy+cDgcyS531+b86T3/meN+/JZmstAACgJ+9Z9AIAAGDWhFwAALoj5AIA0B0hFwCA7gi5AAB0R8gFAKA7Qu5DZOZBZv7io67l8WNWOA9zwnmZFc7DnJxP1yE3M48z8ycXvY63klMfz8wvZubJ6eD90KLXdRE9BrPyh5n5tTMf/52Zbyx6XReNOeG8HoNZ+bnMfPX0ueffM/NuZn7botd10ZiT+eo65D4Gno2IX4iIn4iIJyPi7yPixYWuiKXUWvvl1tq3PPiIiD+JiD9b9LpYLuaEd+HTEfHjrbXLEfF9EXEpIj6+2CWxhB7rObmQITczvz0z/yIzv5yZr59+/oE3fdmHMvMfM/OrmbmfmU+eqf/RzPxMZk4y835mjotL+d6IeLm19rnW2jci4o8j4qnisZiDJZqVs2v65oh4JiLuDj0Ws2FOOK9lmZXW2mutta+c+U/fiIjvrxyL2TMns3EhQ25Mz/uFiLgaEVci4r8i4g/e9DUfi+lV1u+OiP+NiN+PiMjM74mIv4zpK5knI+LXIuJTmfmdb26SmVdOB+zKW6zjkzEd0h/IzCci4kZE/NXAc2O2lmVWznomIr4cEX9XOSHmwpxwXkszK5n54cw8iYg3YjovO8NOjRkyJzNwIUNua+0/Wmufaq39Z2vtjYj4nYi49qYve7G19s+tta9HxG9GxM9m5nsj4iMR8VJr7aXW2v+11v46Ij4bET/9kD5faK2NWmtfeIul/FtEvBwRr8Z0gJ+NiOdmcpLMxBLNylk3IuITrbU26OSYGXPCeS3TrLTWXj79NfQHIuL3IuJ4JifJYOZkNi5kyM3M92fmH2Xm5zPzqzG90jE6HY4HXjvz+ecj4omI+I6Yvqp69vSVzyQzJxHx4Zi+knq3fisifjgiPhgR74uI2xHxt5n5/sKxmIMlmpUH67kSEeOI+ET1GMyeOeG8lm1WIiJaa1+M6W8RPznkOMyOOZmNS4tewIL8akT8YET8SGvtS5m5GhGvRESe+ZoPnvn8SkT8T0R8JaZD9WJr7ZdmsI7ViPjT1tq/nv57NzN3Ynpf7mdncHyGW5ZZeeCjEfHp1trnZnhMhjMnnNeyzcoDlyLiQ3M4LjXmZAYuwpXcJzLzfWc+LkXEt8b09oDJ6Y3aWw+p+0hmPnV6VfW3I+LPz7w57Gcy86cy872nxxw/5Ibw8/inmL7a+q7MfE9mfjSmr8T+pXSmDLXMs/LAxyJid0A9w5kTzmtpZyUzf/7BfZiZeTWmvw7/m+J5Mow5mZOLEHJfiumgPPjYjulN098U01c8/xAPf7PXizF9kvhSTG8l+JWI6TsNI2ItIn4jpm/qeC0ifj0e8r3M6Q3dX8u3vqH7dyPifkQcRcQkpvfjPtNam7z702QGlnlWIjN/LKb3RPmTUItlTjivZZ6VpyLiM5n59Zj+mahXI2IeV/54Z+ZkTtJ7EgAA6M1FuJILAMAFI+QCANAdIRcAgO4IuQAAdGdefyf3kb+bbXd3t1S3vb1d7jkajUp1Ozv1/yPeeDwu1w6Q7/wlJY98Tg4ODkp11fmKiNjb2yvVnZyclHveu3evVDdwvuY1JxELmJX9/f1S3ebm5oxX8s6qcx0RsbKyMrN1vAtLtaccHx+XG1b38yF7SnVvuHz5crnn0dFRqW7gfC3dnjKZ1P/40SJmpbre6uMdsVx7iiu5AAB0R8gFAKA7Qi4AAN0RcgEA6I6QCwBAd4RcAAC6I+QCANAdIRcAgO4IuQAAdEfIBQCgO0IuAADdEXIBAOiOkAsAQHcuLXoBZx0cHJRrb968WapbW1sr9xyNRqW69fX1cs/JZFKuJeLWrVuluiHf942NjVLdnTt3yj2rs9mb4+Pjcu2Qn9NHbW9vr1xb/ZnoySK+B3fv3i3X3rt3r1Q3ZE/x3DM15PtQ/TkdshdVe+7u7pZ7bm9vl2tnzZVcAAC6I+QCANAdIRcAgO4IuQAAdEfIBQCgO0IuAADdEXIBAOiOkAsAQHeEXAAAuiPkAgDQHSEXAIDuCLkAAHRHyAUAoDtCLgAA3cnW2jyOWzrorVu3yg2Pj49LdXt7e+We4/G4VDcajco9h6x3gJzTcecyfG+nOidDHrPDw8NS3Y0bN8o9J5NJuXaAec1JxAJmZWdnp1S3urpa7nn9+vVS3bVr18o9Dw4OyrUDdLOnLEL1ufLo6Kjcs7M5ibggs1LNKUP2sereOdBDZ8WVXAAAuiPkAgDQHSEXAIDuCLkAAHRHyAUAoDtCLgAA3RFyAQDojpALAEB3hFwAALoj5AIA0B0hFwCA7gi5AAB0R8gFAKA7lxa9gLNWVlbKtcfHx6W67e3tcs/Dw8NS3SuvvFLuyTCTyaRUV52viIitra1S3Wg0KvesrnfIz2BvNjY2SnVD9pSq6l4UUV/vIs6TqdXV1VLd7u5uuWd17xyyj/Wmui+vr6/PdiHnsLOz88h7zoMruQAAdEfIBQCgO0IuAADdEXIBAOiOkAsAQHeEXAAAuiPkAgDQHSEXAIDuCLkAAHRHyAUAoDtCLgAA3RFyAQDojpALAEB3hFwAALqTrbV5HHcuB307q6urpbr79++Xe964caNUt7u7W+65IDmn45bmZH9/v9xwfX29XPs42draKtVtb28PaTuvOYkozsrR0VG54Xg8LtWdnJyUe1ZV96KI+mO+srJS7hlLtqdcFEMes+reubOzU+4ZS7inDFHdj6p7UUR9P3rhhRfKPTc2Nsq1Azx0VlzJBQCgO0IuAADdEXIBAOiOkAsAQHeEXAAAuiPkAgDQHSEXAIDuCLkAAHRHyAUAoDtCLgAA3RFyAQDojpALAEB3hFwAALqTrbV5HHcuB307q6urj7pljEajUt2Qte7s7JRrB8g5Hbc0JwcHB+WGe3t7pbqjo6Nyz+Pj40feszqbA81rTiIWMCvXr18v11atra2V6qpzvUBLtadcFOPx+JH3HPIzGEu4p0wmk1mv4x0N2c+rj3n1eWto7QAPnRVXcgEA6I6QCwBAd4RcAAC6I+QCANAdIRcAgO4IuQAAdEfIBQCgO0IuAADdEXIBAOiOkAsAQHeEXAAAuiPkAgDQHSEXAIDuCLkAAHTn0qIXMCuj0ahUNx6Pyz23t7dLddW1LqrnshnymJ2cnJTqdnd3yz3X19dLdT09ZosyZFY2NzdLdXfu3Cn3vHnzZrmWxdjf3y/VXb16tdzz6OjokdZF1J97enN4eFiu3draKtXdvn273HNjY6NUN2Qvmkwmpbp5POe5kgsAQHeEXAAAuiPkAgDQHSEXAIDuCLkAAHRHyAUAoDtCLgAA3RFyAQDojpALAEB3hFwAALoj5AIA0B0hFwCA7gi5AAB059KiFzArzz33XKlufX293PP27dulurW1tXLP0WhUriXi9ddfL9WdnJyUe25sbJRrefw8/fTT5dohewOL8fzzz5fqDg8Pyz0vX75cqhuyF9nHpq5du1auHY/HpbrqjEVETCaTUt3m5ma55zLlFFdyAQDojpALAEB3hFwAALoj5AIA0B0hFwCA7gi5AAB0R8gFAKA7Qi4AAN0RcgEA6I6QCwBAd4RcAAC6I+QCANAdIRcAgO4IuQAAdCdba4teAwAAzJQruQAAdEfIBQCgO0IuAADdEXIBAOiOkAsAQHeEXAAAuiPkAgDQHSEXAIDuCLkAAHRHyAUAoDtCLgAA3RFyAQDojpALAEB3hFwAALoj5AIA0B0hFwCA7gi5AAB0R8gFAKA7Qi4AAN0RcgEA6I6QCwBAd4RcAAC6I+QCANCd/wfBp2WqC3Q/FQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# import necessary packages\n", "import numpy as np\n", @@ -3606,11 +3571,33 @@ }, { "cell_type": "code", - "execution_count": 28, - "metadata": { - "collapsed": false - }, - "outputs": [], + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n" + ] + }, + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'tensorflow'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mto_categorical\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_selection\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtrain_test_split\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;31m# representation of labels\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_categorical\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.7/site-packages/keras/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0m__future__\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mabsolute_import\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mutils\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mactivations\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mapplications\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.7/site-packages/keras/utils/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdata_utils\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mio_utils\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mconv_utils\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;31m# Globally-importable utils.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.7/site-packages/keras/utils/conv_utils.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmoves\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mK\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.7/site-packages/keras/backend/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 87\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0m_BACKEND\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'tensorflow'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstderr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Using TensorFlow backend.\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mtensorflow_backend\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 90\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 91\u001b[0m \u001b[0;31m# Try and load external backend.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0m__future__\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mprint_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframework\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mops\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtf_ops\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmoving_averages\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'tensorflow'" + ] + } + ], "source": [ "from keras.utils import to_categorical\n", "from sklearn.model_selection import train_test_split\n", @@ -3637,11 +3624,21 @@ }, { "cell_type": "code", - "execution_count": 29, - "metadata": { - "collapsed": false - }, - "outputs": [], + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'tensorflow'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mConvolutionalNeuralNetworkTensorflow\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m def __init__(\n", + "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'tensorflow'" + ] + } + ], "source": [ "\n", "import tensorflow as tf\n", @@ -3796,9 +3793,7 @@ { "cell_type": "code", "execution_count": 30, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "epochs = 100\n", @@ -3837,9 +3832,7 @@ { "cell_type": "code", "execution_count": 31, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# visual representation of grid search\n", @@ -3885,9 +3878,7 @@ { "cell_type": "code", "execution_count": 32, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "from keras.models import Sequential\n", @@ -3936,9 +3927,7 @@ { "cell_type": "code", "execution_count": 33, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "CNN_keras = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)\n", @@ -4277,9 +4266,7 @@ { "cell_type": "code", "execution_count": 34, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Note that we use the numpy wrapper for Autograd (see the gradient descent slides)\n", @@ -4363,9 +4350,7 @@ { "cell_type": "code", "execution_count": 35, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# The trial solution using the deep neural network:\n", @@ -4486,9 +4471,7 @@ { "cell_type": "code", "execution_count": 36, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "def solve_ode_neural_network(x, num_neurons_hidden, num_iter, lmb):\n", @@ -4551,9 +4534,7 @@ { "cell_type": "code", "execution_count": 37, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "def deep_neural_network(deep_params, x):\n", @@ -4615,9 +4596,7 @@ { "cell_type": "code", "execution_count": 38, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# The trial solution using the deep neural network:\n", @@ -4698,9 +4677,7 @@ { "cell_type": "code", "execution_count": 39, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "def g_analytic(x, gamma = 2, g0 = 10):\n", @@ -4724,9 +4701,7 @@ { "cell_type": "code", "execution_count": 40, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "npr.seed(15)\n", @@ -4769,9 +4744,7 @@ { "cell_type": "code", "execution_count": 41, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "npr.seed(15)\n", @@ -4820,7 +4793,25 @@ ] } ], - "metadata": {}, + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.0" + } + }, "nbformat": 4, "nbformat_minor": 2 }