{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy import optimize\n",
    "\n",
    "class Neural_Network(object):\n",
    "    def __init__(self, Lambda=0):        \n",
    "        #Define Hyperparameters\n",
    "        self.inputLayerSize = 2\n",
    "        self.outputLayerSize = 1\n",
    "        self.hiddenLayerSize = 3\n",
    "        \n",
    "        #Weights (parameters)\n",
    "        self.W1 = np.random.randn(self.inputLayerSize,self.hiddenLayerSize)\n",
    "        self.W2 = np.random.randn(self.hiddenLayerSize,self.outputLayerSize)\n",
    "        \n",
    "        #Regularization Parameter:\n",
    "        self.Lambda = Lambda\n",
    "        \n",
    "    def forward(self, X):\n",
    "        #Propogate inputs though network\n",
    "        self.z2 = np.dot(X, self.W1)\n",
    "        self.a2 = self.sigmoid(self.z2)\n",
    "        self.z3 = np.dot(self.a2, self.W2)\n",
    "        yHat = self.sigmoid(self.z3) \n",
    "        return yHat\n",
    "        \n",
    "    def sigmoid(self, z):\n",
    "        #Apply sigmoid activation function to scalar, vector, or matrix\n",
    "        return 1/(1+np.exp(-z))\n",
    "    \n",
    "    def sigmoidPrime(self,z):\n",
    "        #Gradient of sigmoid\n",
    "        return np.exp(-z)/((1+np.exp(-z))**2)\n",
    "    \n",
    "    def costFunction(self, X, y):\n",
    "        #Compute cost for given X,y, use weights already stored in class.\n",
    "        self.yHat = self.forward(X)\n",
    "        J = 0.5*sum((y-self.yHat)**2)/X.shape[0] + (self.Lambda/2)*(np.sum(self.W1**2)+np.sum(self.W2**2))\n",
    "        return J\n",
    "        \n",
    "    def costFunctionPrime(self, X, y):\n",
    "        #Compute derivative with respect to W and W2 for a given X and y:\n",
    "        self.yHat = self.forward(X)\n",
    "        \n",
    "        delta3 = np.multiply(-(y-self.yHat), self.sigmoidPrime(self.z3))\n",
    "        #Add gradient of regularization term:\n",
    "        dJdW2 = np.dot(self.a2.T, delta3)/X.shape[0] + self.Lambda*self.W2\n",
    "        \n",
    "        delta2 = np.dot(delta3, self.W2.T)*self.sigmoidPrime(self.z2)\n",
    "        #Add gradient of regularization term:\n",
    "        dJdW1 = np.dot(X.T, delta2)/X.shape[0] + self.Lambda*self.W1\n",
    "        \n",
    "        return dJdW1, dJdW2\n",
    "    \n",
    "    #Helper functions for interacting with other methods/classes\n",
    "    def getParams(self):\n",
    "        #Get W1 and W2 Rolled into vector:\n",
    "        params = np.concatenate((self.W1.ravel(), self.W2.ravel()))\n",
    "        return params\n",
    "    \n",
    "    def setParams(self, params):\n",
    "        #Set W1 and W2 using single parameter vector:\n",
    "        W1_start = 0\n",
    "        W1_end = self.hiddenLayerSize*self.inputLayerSize\n",
    "        self.W1 = np.reshape(params[W1_start:W1_end], \\\n",
    "                             (self.inputLayerSize, self.hiddenLayerSize))\n",
    "        W2_end = W1_end + self.hiddenLayerSize*self.outputLayerSize\n",
    "        self.W2 = np.reshape(params[W1_end:W2_end], \\\n",
    "                             (self.hiddenLayerSize, self.outputLayerSize))\n",
    "        \n",
    "    def computeGradients(self, X, y):\n",
    "        dJdW1, dJdW2 = self.costFunctionPrime(X, y)\n",
    "        return np.concatenate((dJdW1.ravel(), dJdW2.ravel()))\n",
    "    \n",
    "        \n",
    "class trainer(object):\n",
    "    def __init__(self, N):\n",
    "        #Make Local reference to network:\n",
    "        self.N = N\n",
    "        \n",
    "    def callbackF(self, params):\n",
    "        self.N.setParams(params)\n",
    "        self.J.append(self.N.costFunction(self.X, self.y))\n",
    "        self.testJ.append(self.N.costFunction(self.testX, self.testY))\n",
    "        \n",
    "    def costFunctionWrapper(self, params, X, y):\n",
    "        self.N.setParams(params)\n",
    "        cost = self.N.costFunction(X, y)\n",
    "        grad = self.N.computeGradients(X,y)\n",
    "        return cost, grad\n",
    "        \n",
    "    def train(self, trainX, trainY, testX, testY):\n",
    "        #Make an internal variable for the callback function:\n",
    "        self.X = trainX\n",
    "        self.y = trainY\n",
    "        \n",
    "        self.testX = testX\n",
    "        self.testY = testY\n",
    "\n",
    "        #Make empty list to store training costs:\n",
    "        self.J = []\n",
    "        self.testJ = []\n",
    "        \n",
    "        params0 = self.N.getParams()\n",
    "\n",
    "        options = {'maxiter': 200, 'disp' : True}\n",
    "        _res = optimize.minimize(self.costFunctionWrapper, params0, jac=True, method='BFGS', \\\n",
    "                                 args=(trainX, trainY), options=options, callback=self.callbackF)\n",
    "\n",
    "        self.N.setParams(_res.x)\n",
    "        self.optimizationResults = _res\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}