mhjensen
271 lines
12 KiB
Plaintext
271 lines
12 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 33,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import numpy as np\n",
|
|
"import random\n",
|
|
"class Network(object):\n",
|
|
" \n",
|
|
" def _init_(self, sizes):\n",
|
|
" self.num_layers=len(sizes)\n",
|
|
" self.sizes=sizes\n",
|
|
" self.biases=[np.random.randn(y,1) for y in sizes[1:]]\n",
|
|
" self.weights=[np.random.randn(y,x) for x,y in zip(sizes[:-1], sizes[1:])]\n",
|
|
"\n",
|
|
"#sizes is the number of neurons in each layer\n",
|
|
"#for example, say n_1st_layer=3, n_2nd_layer=3, n_3rd_layer=1, then net=Network([3,3,1])\n",
|
|
"\n",
|
|
"#The biases and weights are initialized randomly, using Gaussian distributions of mean=0, stdev=1\n",
|
|
"#z is a vector (or a np.array)\n",
|
|
"\n",
|
|
" def feedforward(self,a):\n",
|
|
" #returns output w/ 'a' as an input\n",
|
|
" for b, w in zip(self.biases, self.weights):\n",
|
|
" a=sigmoid(np.dot(w,b)+b)\n",
|
|
" return a\n",
|
|
" \n",
|
|
"#Apply a Stochastic Gradient Descent (SGD) method:\n",
|
|
" def SGD(self, training_data, epochs, mini_batch_size, eta, test_data=None):\n",
|
|
" \"\"\"Trains network using batches incorporating SGD. The network will be evaluated against the\n",
|
|
" test data after each epoch, with partial progress being printed out (this is useful for tracking,\n",
|
|
" but slows the process.)\"\"\"\n",
|
|
" if test_data: n_test=len(test_data)\n",
|
|
" n=len(training_data)\n",
|
|
" for j in xrange(epochs):\n",
|
|
" random.shuffle(training_data)\n",
|
|
" mini_batches=[training_data[k:k+mini_batch_size] for k in xrange(o,n,mini_batch_size)]\n",
|
|
" for mini_batch in mini_batches:\n",
|
|
" self.update_mini_batch(mini_batch, eta)\n",
|
|
" if test_data:\n",
|
|
" print (\"Epoch {0}: {1}/{2}\".format(j, self.evaluate(test_data), n_test))\n",
|
|
" else:\n",
|
|
" print (\"Epoch {0} complete\".format(j))\n",
|
|
" \n",
|
|
" \n",
|
|
" def update_mini_batch(self, mini_batch, eta):\n",
|
|
" #updates w and b using backpropagation to a single mini batch. eta is the learning rate.\"\n",
|
|
" nabla_b=[np.zeros(b.shape) for b in self.biases]\n",
|
|
" nabla_w=[np.zeros(w.shape) for w in self.weights]\n",
|
|
" for x,y in mini_batch:\n",
|
|
" delta_nabla_b, delta_nabla_w=self.backprop(x,y)\n",
|
|
" nabla_b=[nb+dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]\n",
|
|
" nabla_w=[nw+dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]\n",
|
|
" self.weights=[w-(eta/len(mini_batch))*nw for w, nw in zip(self.weights, nabla_w)]\n",
|
|
" self.biases=[b-(eta/len(mini_batch))*nb for b, nb in zip(self.biases, nabla_b)]\n",
|
|
" \n",
|
|
" def backprop(self, x, y):\n",
|
|
" \"\"\"Return a tuple ``(nabla_b, nabla_w)`` representing the\n",
|
|
" gradient for the cost function C_x. ``nabla_b`` and\n",
|
|
" ``nabla_w`` are layer-by-layer lists of numpy arrays, similar\n",
|
|
" to ``self.biases`` and ``self.weights``.\"\"\"\n",
|
|
" nabla_b = [np.zeros(b.shape) for b in self.biases]\n",
|
|
" nabla_w = [np.zeros(w.shape) for w in self.weights]\n",
|
|
" # feedforward\n",
|
|
" activation = x\n",
|
|
" activations = [x] # list to store all the activations, layer by layer\n",
|
|
" zs = [] # list to store all the z vectors, layer by layer\n",
|
|
" for b, w in zip(self.biases, self.weights):\n",
|
|
" z = np.dot(w, activation)+b\n",
|
|
" zs.append(z)\n",
|
|
" activation = sigmoid(z)\n",
|
|
" activations.append(activation)\n",
|
|
" # backward pass\n",
|
|
" delta = self.cost_derivative(activations[-1], y) * \\\n",
|
|
" sigmoid_prime(zs[-1])\n",
|
|
" nabla_b[-1] = delta\n",
|
|
" nabla_w[-1] = np.dot(delta, activations[-2].transpose())\n",
|
|
" # Note that the variable l in the loop below is used a little\n",
|
|
" # differently to the notation in Chapter 2 of the book. Here,\n",
|
|
" # l = 1 means the last layer of neurons, l = 2 is the\n",
|
|
" # second-last layer, and so on. It's a renumbering of the\n",
|
|
" # scheme in the book, used here to take advantage of the fact\n",
|
|
" # that Python can use negative indices in lists.\n",
|
|
" for l in xrange(2, self.num_layers):\n",
|
|
" z = zs[-l]\n",
|
|
" sp = sigmoid_prime(z)\n",
|
|
" delta = np.dot(self.weights[-l+1].transpose(), delta) * sp\n",
|
|
" nabla_b[-l] = delta\n",
|
|
" nabla_w[-l] = np.dot(delta, activations[-l-1].transpose())\n",
|
|
" return (nabla_b, nabla_w)\n",
|
|
"\n",
|
|
" def evaluate(self, test_data):\n",
|
|
" \"\"\"Return the number of test inputs for which the neural\n",
|
|
" network outputs the correct result. Note that the neural\n",
|
|
" network's output is assumed to be the index of whichever\n",
|
|
" neuron in the final layer has the highest activation.\"\"\"\n",
|
|
" test_results = [(np.argmax(self.feedforward(x)), y)\n",
|
|
" for (x, y) in test_data]\n",
|
|
" return sum(int(x == y) for (x, y) in test_results)\n",
|
|
"\n",
|
|
" def cost_derivative(self, output_activations, y):\n",
|
|
" \"\"\"Return the vector of partial derivatives \\partial C_x /\n",
|
|
" \\partial a for the output activations.\"\"\"\n",
|
|
" return (output_activations-y)\n",
|
|
" \n",
|
|
" \n",
|
|
" \n",
|
|
"#Functions\n",
|
|
"def sigmoid(z):\n",
|
|
" return 1.0/(1.0+np.exp(-z))\n",
|
|
"\n",
|
|
"def sigmoid_prime(z):\n",
|
|
" return sigmoid(z)*(1-sigmoid(z))\n",
|
|
"\n",
|
|
"network=Network()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 35,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"ename": "AttributeError",
|
|
"evalue": "'Network' object has no attribute 'Network'",
|
|
"output_type": "error",
|
|
"traceback": [
|
|
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
|
|
"\u001b[0;32m<ipython-input-35-768076401f8b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 86\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m \u001b[0mnet\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnetwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNetwork\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m784\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 89\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSGD\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraining_data\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtest_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtest_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
|
"\u001b[0;31mAttributeError\u001b[0m: 'Network' object has no attribute 'Network'"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# %load neural-networks-and-deep-learning/src/mnist_loader.py\n",
|
|
"\"\"\"\n",
|
|
"mnist_loader\n",
|
|
"~~~~~~~~~~~~\n",
|
|
"\n",
|
|
"A library to load the MNIST image data. For details of the data\n",
|
|
"structures that are returned, see the doc strings for ``load_data``\n",
|
|
"and ``load_data_wrapper``. In practice, ``load_data_wrapper`` is the\n",
|
|
"function usually called by our neural network code.\n",
|
|
"\"\"\"\n",
|
|
"\n",
|
|
"#### Libraries\n",
|
|
"# Standard library\n",
|
|
"import pickle\n",
|
|
"import gzip\n",
|
|
"\n",
|
|
"# Third-party libraries\n",
|
|
"import numpy as np\n",
|
|
"\n",
|
|
"def load_data():\n",
|
|
" \"\"\"Return the MNIST data as a tuple containing the training data,\n",
|
|
" the validation data, and the test data.\n",
|
|
"\n",
|
|
" The ``training_data`` is returned as a tuple with two entries.\n",
|
|
" The first entry contains the actual training images. This is a\n",
|
|
" numpy ndarray with 50,000 entries. Each entry is, in turn, a\n",
|
|
" numpy ndarray with 784 values, representing the 28 * 28 = 784\n",
|
|
" pixels in a single MNIST image.\n",
|
|
"\n",
|
|
" The second entry in the ``training_data`` tuple is a numpy ndarray\n",
|
|
" containing 50,000 entries. Those entries are just the digit\n",
|
|
" values (0...9) for the corresponding images contained in the first\n",
|
|
" entry of the tuple.\n",
|
|
"\n",
|
|
" The ``validation_data`` and ``test_data`` are similar, except\n",
|
|
" each contains only 10,000 images.\n",
|
|
"\n",
|
|
" This is a nice data format, but for use in neural networks it's\n",
|
|
" helpful to modify the format of the ``training_data`` a little.\n",
|
|
" That's done in the wrapper function ``load_data_wrapper()``, see\n",
|
|
" below.\n",
|
|
" \"\"\"\n",
|
|
" f = gzip.open('../data/mnist.pkl.gz', 'rb')\n",
|
|
" training_data, validation_data, test_data = cPickle.load(f)\n",
|
|
" f.close()\n",
|
|
" return (training_data, validation_data, test_data)\n",
|
|
"\n",
|
|
"def load_data_wrapper():\n",
|
|
" \"\"\"Return a tuple containing ``(training_data, validation_data,\n",
|
|
" test_data)``. Based on ``load_data``, but the format is more\n",
|
|
" convenient for use in our implementation of neural networks.\n",
|
|
"\n",
|
|
" In particular, ``training_data`` is a list containing 50,000\n",
|
|
" 2-tuples ``(x, y)``. ``x`` is a 784-dimensional numpy.ndarray\n",
|
|
" containing the input image. ``y`` is a 10-dimensional\n",
|
|
" numpy.ndarray representing the unit vector corresponding to the\n",
|
|
" correct digit for ``x``.\n",
|
|
"\n",
|
|
" ``validation_data`` and ``test_data`` are lists containing 10,000\n",
|
|
" 2-tuples ``(x, y)``. In each case, ``x`` is a 784-dimensional\n",
|
|
" numpy.ndarry containing the input image, and ``y`` is the\n",
|
|
" corresponding classification, i.e., the digit values (integers)\n",
|
|
" corresponding to ``x``.\n",
|
|
"\n",
|
|
" Obviously, this means we're using slightly different formats for\n",
|
|
" the training data and the validation / test data. These formats\n",
|
|
" turn out to be the most convenient for use in our neural network\n",
|
|
" code.\"\"\"\n",
|
|
" tr_d, va_d, te_d = load_data()\n",
|
|
" training_inputs = [np.reshape(x, (784, 1)) for x in tr_d[0]]\n",
|
|
" training_results = [vectorized_result(y) for y in tr_d[1]]\n",
|
|
" training_data = zip(training_inputs, training_results)\n",
|
|
" validation_inputs = [np.reshape(x, (784, 1)) for x in va_d[0]]\n",
|
|
" validation_data = zip(validation_inputs, va_d[1])\n",
|
|
" test_inputs = [np.reshape(x, (784, 1)) for x in te_d[0]]\n",
|
|
" test_data = zip(test_inputs, te_d[1])\n",
|
|
" return (training_data, validation_data, test_data)\n",
|
|
"\n",
|
|
"def vectorized_result(j):\n",
|
|
" \"\"\"Return a 10-dimensional unit vector with a 1.0 in the jth\n",
|
|
" position and zeroes elsewhere. This is used to convert a digit\n",
|
|
" (0...9) into a corresponding desired output from the neural\n",
|
|
" network.\"\"\"\n",
|
|
" e = np.zeros((10, 1))\n",
|
|
" e[j] = 1.0\n",
|
|
" return e\n",
|
|
"\n",
|
|
"net=network.Network([784,30,30])\n",
|
|
"net.SGD(training_data,30,10,3,test_data=test_data)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.6.3"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|