Updating various chapters with codes
This commit is contained in:
mhjensen
@@ -394,6 +394,433 @@ it has been shown that the hyperbolic tangent
|
||||
performs better than the sigmoid for training MLPs.
|
||||
has become the most popular for *deep neural networks*
|
||||
|
||||
!bc pycod
|
||||
"""The sigmoid function (or the logistic curve) is a
|
||||
function that takes any real number, z, and outputs a number (0,1).
|
||||
It is useful in neural networks for assigning weights on a relative scale.
|
||||
The value z is the weighted sum of parameters involved in the learning algorithm."""
|
||||
|
||||
!split
|
||||
===
|
||||
import numpy
|
||||
import matplotlib.pyplot as plt
|
||||
import math as mt
|
||||
|
||||
z = numpy.arange(-5, 5, .1)
|
||||
sigma_fn = numpy.vectorize(lambda z: 1/(1+numpy.exp(-z)))
|
||||
sigma = sigma_fn(z)
|
||||
|
||||
fig = plt.figure()
|
||||
ax = fig.add_subplot(111)
|
||||
ax.plot(z, sigma)
|
||||
ax.set_ylim([-0.1, 1.1])
|
||||
ax.set_xlim([-5,5])
|
||||
ax.grid(True)
|
||||
ax.set_xlabel('z')
|
||||
ax.set_title('sigmoid function')
|
||||
|
||||
plt.show()
|
||||
|
||||
"""Step Function"""
|
||||
z = numpy.arange(-5, 5, .02)
|
||||
step_fn = numpy.vectorize(lambda z: 1.0 if z >= 0.0 else 0.0)
|
||||
step = step_fn(z)
|
||||
|
||||
fig = plt.figure()
|
||||
ax = fig.add_subplot(111)
|
||||
ax.plot(z, step)
|
||||
ax.set_ylim([-0.5, 1.5])
|
||||
ax.set_xlim([-5,5])
|
||||
ax.grid(True)
|
||||
ax.set_xlabel('z')
|
||||
ax.set_title('step function')
|
||||
|
||||
plt.show()
|
||||
|
||||
"""Sine Function"""
|
||||
z = numpy.arange(-2*mt.pi, 2*mt.pi, 0.1)
|
||||
t = numpy.sin(z)
|
||||
|
||||
fig = plt.figure()
|
||||
ax = fig.add_subplot(111)
|
||||
ax.plot(z, t)
|
||||
ax.set_ylim([-1.0, 1.0])
|
||||
ax.set_xlim([-2*mt.pi,2*mt.pi])
|
||||
ax.grid(True)
|
||||
ax.set_xlabel('z')
|
||||
ax.set_title('sine function')
|
||||
|
||||
plt.show()
|
||||
|
||||
"""Plots a graph of the squashing function used by a rectified linear
|
||||
unit"""
|
||||
z = numpy.arange(-2, 2, .1)
|
||||
zero = numpy.zeros(len(z))
|
||||
y = numpy.max([zero, z], axis=0)
|
||||
|
||||
fig = plt.figure()
|
||||
ax = fig.add_subplot(111)
|
||||
ax.plot(z, y)
|
||||
ax.set_ylim([-2.0, 2.0])
|
||||
ax.set_xlim([-2.0, 2.0])
|
||||
ax.grid(True)
|
||||
ax.set_xlabel('z')
|
||||
ax.set_title('Rectified linear unit')
|
||||
|
||||
plt.show()
|
||||
!ec
|
||||
|
||||
|
||||
!bc pycod
|
||||
from scipy import optimize
|
||||
|
||||
class Neural_Network(object):
|
||||
def __init__(self, Lambda=0):
|
||||
#Define Hyperparameters
|
||||
self.inputLayerSize = 2
|
||||
self.outputLayerSize = 1
|
||||
self.hiddenLayerSize = 3
|
||||
|
||||
#Weights (parameters)
|
||||
self.W1 = np.random.randn(self.inputLayerSize,self.hiddenLayerSize)
|
||||
self.W2 = np.random.randn(self.hiddenLayerSize,self.outputLayerSize)
|
||||
|
||||
#Regularization Parameter:
|
||||
self.Lambda = Lambda
|
||||
|
||||
def forward(self, X):
|
||||
#Propogate inputs though network
|
||||
self.z2 = np.dot(X, self.W1)
|
||||
self.a2 = self.sigmoid(self.z2)
|
||||
self.z3 = np.dot(self.a2, self.W2)
|
||||
yHat = self.sigmoid(self.z3)
|
||||
return yHat
|
||||
|
||||
def sigmoid(self, z):
|
||||
#Apply sigmoid activation function to scalar, vector, or matrix
|
||||
return 1/(1+np.exp(-z))
|
||||
|
||||
def sigmoidPrime(self,z):
|
||||
#Gradient of sigmoid
|
||||
return np.exp(-z)/((1+np.exp(-z))**2)
|
||||
|
||||
def costFunction(self, X, y):
|
||||
#Compute cost for given X,y, use weights already stored in class.
|
||||
self.yHat = self.forward(X)
|
||||
J = 0.5*sum((y-self.yHat)**2)/X.shape[0] + (self.Lambda/2)*(np.sum(self.W1**2)+np.sum(self.W2**2))
|
||||
return J
|
||||
|
||||
def costFunctionPrime(self, X, y):
|
||||
#Compute derivative with respect to W and W2 for a given X and y:
|
||||
self.yHat = self.forward(X)
|
||||
|
||||
delta3 = np.multiply(-(y-self.yHat), self.sigmoidPrime(self.z3))
|
||||
#Add gradient of regularization term:
|
||||
dJdW2 = np.dot(self.a2.T, delta3)/X.shape[0] + self.Lambda*self.W2
|
||||
|
||||
delta2 = np.dot(delta3, self.W2.T)*self.sigmoidPrime(self.z2)
|
||||
#Add gradient of regularization term:
|
||||
dJdW1 = np.dot(X.T, delta2)/X.shape[0] + self.Lambda*self.W1
|
||||
|
||||
return dJdW1, dJdW2
|
||||
|
||||
#Helper functions for interacting with other methods/classes
|
||||
def getParams(self):
|
||||
#Get W1 and W2 Rolled into vector:
|
||||
params = np.concatenate((self.W1.ravel(), self.W2.ravel()))
|
||||
return params
|
||||
|
||||
def setParams(self, params):
|
||||
#Set W1 and W2 using single parameter vector:
|
||||
W1_start = 0
|
||||
W1_end = self.hiddenLayerSize*self.inputLayerSize
|
||||
self.W1 = np.reshape(params[W1_start:W1_end], \
|
||||
(self.inputLayerSize, self.hiddenLayerSize))
|
||||
W2_end = W1_end + self.hiddenLayerSize*self.outputLayerSize
|
||||
self.W2 = np.reshape(params[W1_end:W2_end], \
|
||||
(self.hiddenLayerSize, self.outputLayerSize))
|
||||
|
||||
def computeGradients(self, X, y):
|
||||
dJdW1, dJdW2 = self.costFunctionPrime(X, y)
|
||||
return np.concatenate((dJdW1.ravel(), dJdW2.ravel()))
|
||||
|
||||
|
||||
class trainer(object):
|
||||
def __init__(self, N):
|
||||
#Make Local reference to network:
|
||||
self.N = N
|
||||
|
||||
def callbackF(self, params):
|
||||
self.N.setParams(params)
|
||||
self.J.append(self.N.costFunction(self.X, self.y))
|
||||
self.testJ.append(self.N.costFunction(self.testX, self.testY))
|
||||
|
||||
def costFunctionWrapper(self, params, X, y):
|
||||
self.N.setParams(params)
|
||||
cost = self.N.costFunction(X, y)
|
||||
grad = self.N.computeGradients(X,y)
|
||||
return cost, grad
|
||||
|
||||
def train(self, trainX, trainY, testX, testY):
|
||||
#Make an internal variable for the callback function:
|
||||
self.X = trainX
|
||||
self.y = trainY
|
||||
|
||||
self.testX = testX
|
||||
self.testY = testY
|
||||
|
||||
#Make empty list to store training costs:
|
||||
self.J = []
|
||||
self.testJ = []
|
||||
|
||||
params0 = self.N.getParams()
|
||||
|
||||
options = {'maxiter': 200, 'disp' : True}
|
||||
_res = optimize.minimize(self.costFunctionWrapper, params0, jac=True, method='BFGS', \
|
||||
args=(trainX, trainY), options=options, callback=self.callbackF)
|
||||
|
||||
self.N.setParams(_res.x)
|
||||
self.optimizationResults = _res
|
||||
!ec
|
||||
|
||||
|
||||
!split
|
||||
===== Two-layer Neural Network =====
|
||||
!bc pycod
|
||||
import numpy as np
|
||||
|
||||
#sigmoid
|
||||
def nonlin(x, deriv=False):
|
||||
if (deriv==True):
|
||||
return x*(1-x)
|
||||
return 1/(1+np.exp(-x))
|
||||
|
||||
#input data
|
||||
x=np.array([[0,0,1],[0,1,1],[1,0,1],[1,1,1]])
|
||||
|
||||
#output data
|
||||
y=np.array([0,1,1,0]).T
|
||||
|
||||
#seed random numbers to make calculation
|
||||
np.random.seed(1)
|
||||
|
||||
#initialize weights with mean=0
|
||||
syn0=2*np.random.random((3,4))-1
|
||||
|
||||
for iter in range(10000):
|
||||
#forward propogation
|
||||
l0=x
|
||||
l1=nonlin(np.dot(l0,syn0))
|
||||
l1_error=y-l1
|
||||
#multiply error by slope of sigmoid at values of l1
|
||||
l1_delta=l1_error*nonlin(l1,True)
|
||||
#update weights
|
||||
syn0+=np.dot(l0.T, l1_delta)
|
||||
|
||||
print("Output after training: ",l1 )
|
||||
!ec
|
||||
|
||||
|
||||
!bc pycod
|
||||
import numpy as np
|
||||
import random
|
||||
class Network(object):
|
||||
|
||||
def _init_(self, sizes):
|
||||
self.num_layers=len(sizes)
|
||||
self.sizes=sizes
|
||||
self.biases=[np.random.randn(y,1) for y in sizes[1:]]
|
||||
self.weights=[np.random.randn(y,x) for x,y in zip(sizes[:-1], sizes[1:])]
|
||||
|
||||
#sizes is the number of neurons in each layer
|
||||
#for example, say n_1st_layer=3, n_2nd_layer=3, n_3rd_layer=1, then net=Network([3,3,1])
|
||||
|
||||
#The biases and weights are initialized randomly, using Gaussian distributions of mean=0, stdev=1
|
||||
#z is a vector (or a np.array)
|
||||
|
||||
def feedforward(self,a):
|
||||
#returns output w/ 'a' as an input
|
||||
for b, w in zip(self.biases, self.weights):
|
||||
a=sigmoid(np.dot(w,b)+b)
|
||||
return a
|
||||
|
||||
#Apply a Stochastic Gradient Descent (SGD) method:
|
||||
def SGD(self, training_data, epochs, mini_batch_size, eta, test_data=None):
|
||||
"""Trains network using batches incorporating SGD. The network will be evaluated against the
|
||||
test data after each epoch, with partial progress being printed out (this is useful for tracking,
|
||||
but slows the process.)"""
|
||||
if test_data: n_test=len(test_data)
|
||||
n=len(training_data)
|
||||
for j in xrange(epochs):
|
||||
random.shuffle(training_data)
|
||||
mini_batches=[training_data[k:k+mini_batch_size] for k in xrange(o,n,mini_batch_size)]
|
||||
for mini_batch in mini_batches:
|
||||
self.update_mini_batch(mini_batch, eta)
|
||||
if test_data:
|
||||
print ("Epoch {0}: {1}/{2}".format(j, self.evaluate(test_data), n_test))
|
||||
else:
|
||||
print ("Epoch {0} complete".format(j))
|
||||
|
||||
|
||||
def update_mini_batch(self, mini_batch, eta):
|
||||
#updates w and b using backpropagation to a single mini batch. eta is the learning rate."
|
||||
nabla_b=[np.zeros(b.shape) for b in self.biases]
|
||||
nabla_w=[np.zeros(w.shape) for w in self.weights]
|
||||
for x,y in mini_batch:
|
||||
delta_nabla_b, delta_nabla_w=self.backprop(x,y)
|
||||
nabla_b=[nb+dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]
|
||||
nabla_w=[nw+dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]
|
||||
self.weights=[w-(eta/len(mini_batch))*nw for w, nw in zip(self.weights, nabla_w)]
|
||||
self.biases=[b-(eta/len(mini_batch))*nb for b, nb in zip(self.biases, nabla_b)]
|
||||
|
||||
def backprop(self, x, y):
|
||||
"""Return a tuple ``(nabla_b, nabla_w)`` representing the
|
||||
gradient for the cost function C_x. ``nabla_b`` and
|
||||
``nabla_w`` are layer-by-layer lists of numpy arrays, similar
|
||||
to ``self.biases`` and ``self.weights``."""
|
||||
nabla_b = [np.zeros(b.shape) for b in self.biases]
|
||||
nabla_w = [np.zeros(w.shape) for w in self.weights]
|
||||
# feedforward
|
||||
activation = x
|
||||
activations = [x] # list to store all the activations, layer by layer
|
||||
zs = [] # list to store all the z vectors, layer by layer
|
||||
for b, w in zip(self.biases, self.weights):
|
||||
z = np.dot(w, activation)+b
|
||||
zs.append(z)
|
||||
activation = sigmoid(z)
|
||||
activations.append(activation)
|
||||
# backward pass
|
||||
delta = self.cost_derivative(activations[-1], y) * \
|
||||
sigmoid_prime(zs[-1])
|
||||
nabla_b[-1] = delta
|
||||
nabla_w[-1] = np.dot(delta, activations[-2].transpose())
|
||||
# Note that the variable l in the loop below is used a little
|
||||
# differently to the notation in Chapter 2 of the book. Here,
|
||||
# l = 1 means the last layer of neurons, l = 2 is the
|
||||
# second-last layer, and so on. It's a renumbering of the
|
||||
# scheme in the book, used here to take advantage of the fact
|
||||
# that Python can use negative indices in lists.
|
||||
for l in xrange(2, self.num_layers):
|
||||
z = zs[-l]
|
||||
sp = sigmoid_prime(z)
|
||||
delta = np.dot(self.weights[-l+1].transpose(), delta) * sp
|
||||
nabla_b[-l] = delta
|
||||
nabla_w[-l] = np.dot(delta, activations[-l-1].transpose())
|
||||
return (nabla_b, nabla_w)
|
||||
|
||||
def evaluate(self, test_data):
|
||||
"""Return the number of test inputs for which the neural
|
||||
network outputs the correct result. Note that the neural
|
||||
network's output is assumed to be the index of whichever
|
||||
neuron in the final layer has the highest activation."""
|
||||
test_results = [(np.argmax(self.feedforward(x)), y)
|
||||
for (x, y) in test_data]
|
||||
return sum(int(x == y) for (x, y) in test_results)
|
||||
|
||||
def cost_derivative(self, output_activations, y):
|
||||
"""Return the vector of partial derivatives \partial C_x /
|
||||
\partial a for the output activations."""
|
||||
return (output_activations-y)
|
||||
|
||||
|
||||
|
||||
#Functions
|
||||
def sigmoid(z):
|
||||
return 1.0/(1.0+np.exp(-z))
|
||||
|
||||
def sigmoid_prime(z):
|
||||
return sigmoid(z)*(1-sigmoid(z))
|
||||
|
||||
network=Network()
|
||||
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
# %load neural-networks-and-deep-learning/src/mnist_loader.py
|
||||
"""
|
||||
mnist_loader
|
||||
~~~~~~~~~~~~
|
||||
|
||||
A library to load the MNIST image data. For details of the data
|
||||
structures that are returned, see the doc strings for ``load_data``
|
||||
and ``load_data_wrapper``. In practice, ``load_data_wrapper`` is the
|
||||
function usually called by our neural network code.
|
||||
"""
|
||||
|
||||
#### Libraries
|
||||
# Standard library
|
||||
import pickle
|
||||
import gzip
|
||||
|
||||
# Third-party libraries
|
||||
import numpy as np
|
||||
|
||||
def load_data():
|
||||
"""Return the MNIST data as a tuple containing the training data,
|
||||
the validation data, and the test data.
|
||||
|
||||
The ``training_data`` is returned as a tuple with two entries.
|
||||
The first entry contains the actual training images. This is a
|
||||
numpy ndarray with 50,000 entries. Each entry is, in turn, a
|
||||
numpy ndarray with 784 values, representing the 28 * 28 = 784
|
||||
pixels in a single MNIST image.
|
||||
|
||||
The second entry in the ``training_data`` tuple is a numpy ndarray
|
||||
containing 50,000 entries. Those entries are just the digit
|
||||
values (0...9) for the corresponding images contained in the first
|
||||
entry of the tuple.
|
||||
|
||||
The ``validation_data`` and ``test_data`` are similar, except
|
||||
each contains only 10,000 images.
|
||||
|
||||
This is a nice data format, but for use in neural networks it's
|
||||
helpful to modify the format of the ``training_data`` a little.
|
||||
That's done in the wrapper function ``load_data_wrapper()``, see
|
||||
below.
|
||||
"""
|
||||
f = gzip.open('../data/mnist.pkl.gz', 'rb')
|
||||
training_data, validation_data, test_data = cPickle.load(f)
|
||||
f.close()
|
||||
return (training_data, validation_data, test_data)
|
||||
|
||||
def load_data_wrapper():
|
||||
"""Return a tuple containing ``(training_data, validation_data,
|
||||
test_data)``. Based on ``load_data``, but the format is more
|
||||
convenient for use in our implementation of neural networks.
|
||||
|
||||
In particular, ``training_data`` is a list containing 50,000
|
||||
2-tuples ``(x, y)``. ``x`` is a 784-dimensional numpy.ndarray
|
||||
containing the input image. ``y`` is a 10-dimensional
|
||||
numpy.ndarray representing the unit vector corresponding to the
|
||||
correct digit for ``x``.
|
||||
|
||||
``validation_data`` and ``test_data`` are lists containing 10,000
|
||||
2-tuples ``(x, y)``. In each case, ``x`` is a 784-dimensional
|
||||
numpy.ndarry containing the input image, and ``y`` is the
|
||||
corresponding classification, i.e., the digit values (integers)
|
||||
corresponding to ``x``.
|
||||
|
||||
Obviously, this means we're using slightly different formats for
|
||||
the training data and the validation / test data. These formats
|
||||
turn out to be the most convenient for use in our neural network
|
||||
code."""
|
||||
tr_d, va_d, te_d = load_data()
|
||||
training_inputs = [np.reshape(x, (784, 1)) for x in tr_d[0]]
|
||||
training_results = [vectorized_result(y) for y in tr_d[1]]
|
||||
training_data = zip(training_inputs, training_results)
|
||||
validation_inputs = [np.reshape(x, (784, 1)) for x in va_d[0]]
|
||||
validation_data = zip(validation_inputs, va_d[1])
|
||||
test_inputs = [np.reshape(x, (784, 1)) for x in te_d[0]]
|
||||
test_data = zip(test_inputs, te_d[1])
|
||||
return (training_data, validation_data, test_data)
|
||||
|
||||
def vectorized_result(j):
|
||||
"""Return a 10-dimensional unit vector with a 1.0 in the jth
|
||||
position and zeroes elsewhere. This is used to convert a digit
|
||||
(0...9) into a corresponding desired output from the neural
|
||||
network."""
|
||||
e = np.zeros((10, 1))
|
||||
e[j] = 1.0
|
||||
return e
|
||||
|
||||
net=network.Network([784,30,30])
|
||||
net.SGD(training_data,30,10,3,test_data=test_data)
|
||||
!ec
|
||||
|
||||
Reference in New Issue
Block a user