diff --git a/doc/LectureNotes/_build/.doctrees/environment.pickle b/doc/LectureNotes/_build/.doctrees/environment.pickle index a98aad57d..ea91ee4d9 100644 Binary files a/doc/LectureNotes/_build/.doctrees/environment.pickle and b/doc/LectureNotes/_build/.doctrees/environment.pickle differ diff --git a/doc/LectureNotes/_build/.doctrees/exercisesweek43.doctree b/doc/LectureNotes/_build/.doctrees/exercisesweek43.doctree index 19e7549f9..b3dca6c16 100644 Binary files a/doc/LectureNotes/_build/.doctrees/exercisesweek43.doctree and b/doc/LectureNotes/_build/.doctrees/exercisesweek43.doctree differ diff --git a/doc/LectureNotes/_build/.doctrees/week43.doctree b/doc/LectureNotes/_build/.doctrees/week43.doctree index 60ed8df0a..9933a7f90 100644 Binary files a/doc/LectureNotes/_build/.doctrees/week43.doctree and b/doc/LectureNotes/_build/.doctrees/week43.doctree differ diff --git a/doc/LectureNotes/_build/html/_images/exercisesweek43_11_2.png b/doc/LectureNotes/_build/html/_images/exercisesweek43_11_2.png new file mode 100644 index 000000000..d6998a4a8 Binary files /dev/null and b/doc/LectureNotes/_build/html/_images/exercisesweek43_11_2.png differ diff --git a/doc/LectureNotes/_build/html/_sources/exercisesweek43.ipynb b/doc/LectureNotes/_build/html/_sources/exercisesweek43.ipynb index e84a225b3..e5f90421a 100644 --- a/doc/LectureNotes/_build/html/_sources/exercisesweek43.ipynb +++ b/doc/LectureNotes/_build/html/_sources/exercisesweek43.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "20ae416e", + "id": "60ef4ca3", "metadata": { "editable": true }, @@ -14,13 +14,13 @@ }, { "cell_type": "markdown", - "id": "8bfcd25c", + "id": "6584f2f2", "metadata": { "editable": true }, "source": [ "# Exercises weeks 43 and 44 \n", - "**October 9-13, 2023**\n", + "**October 23-27, 2023**\n", "\n", "Date: **Deadline is Sunday November 5 at midnight**\n", "\n", @@ -29,7 +29,7 @@ }, { "cell_type": "markdown", - "id": "fed7dbe4", + "id": "f5f930b9", "metadata": { "editable": true }, @@ -69,7 +69,7 @@ }, { "cell_type": "markdown", - "id": "f108a242", + "id": "a9d8df6c", "metadata": { "editable": true }, @@ -107,7 +107,7 @@ }, { "cell_type": "markdown", - "id": "aa6993a7", + "id": "925a54fa", "metadata": { "editable": true }, @@ -119,7 +119,7 @@ }, { "cell_type": "markdown", - "id": "90bd0efa", + "id": "1f3c8e67", "metadata": { "editable": true }, @@ -134,7 +134,7 @@ }, { "cell_type": "markdown", - "id": "23ba74e1", + "id": "66ebe4a2", "metadata": { "editable": true }, @@ -161,6 +161,1446 @@ "\n", "Everything you develop here can be used directly into the code for the project." ] + }, + { + "cell_type": "markdown", + "id": "454750aa", + "metadata": { + "editable": true + }, + "source": [ + "## Setting up the Neural Network\n", + "\n", + "We define first our design matrix and the various output vectors for the different gates." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "8308d3ec", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "\"\"\"\n", + "Simple code that tests XOR, OR and AND gates with linear regression\n", + "\"\"\"\n", + "\n", + "# import necessary packages\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from sklearn import datasets\n", + "\n", + "def sigmoid(x):\n", + " return 1/(1 + np.exp(-x))\n", + "\n", + "def feed_forward(X):\n", + " # weighted sum of inputs to the hidden layer\n", + " z_h = np.matmul(X, hidden_weights) + hidden_bias\n", + " # activation in the hidden layer\n", + " a_h = sigmoid(z_h)\n", + " \n", + " # weighted sum of inputs to the output layer\n", + " z_o = np.matmul(a_h, output_weights) + output_bias\n", + " # softmax output\n", + " # axis 0 holds each input and axis 1 the probabilities of each category\n", + " probabilities = sigmoid(z_o)\n", + " return probabilities\n", + "\n", + "# we obtain a prediction by taking the class with the highest likelihood\n", + "def predict(X):\n", + " probabilities = feed_forward(X)\n", + " return np.argmax(probabilities, axis=1)\n", + "\n", + "# ensure the same random numbers appear every time\n", + "np.random.seed(0)\n", + "\n", + "# Design matrix\n", + "X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)\n", + "\n", + "# The XOR gate\n", + "yXOR = np.array( [ 0, 1 ,1, 0])\n", + "# The OR gate\n", + "yOR = np.array( [ 0, 1 ,1, 1])\n", + "# The AND gate\n", + "yAND = np.array( [ 0, 0 ,0, 1])\n", + "\n", + "# Defining the neural network\n", + "n_inputs, n_features = X.shape\n", + "n_hidden_neurons = 2\n", + "n_categories = 2\n", + "n_features = 2\n", + "\n", + "# we make the weights normally distributed using numpy.random.randn\n", + "\n", + "# weights and bias in the hidden layer\n", + "hidden_weights = np.random.randn(n_features, n_hidden_neurons)\n", + "hidden_bias = np.zeros(n_hidden_neurons) + 0.01\n", + "\n", + "# weights and bias in the output layer\n", + "output_weights = np.random.randn(n_hidden_neurons, n_categories)\n", + "output_bias = np.zeros(n_categories) + 0.01\n", + "\n", + "probabilities = feed_forward(X)\n", + "print(probabilities)\n", + "\n", + "\n", + "predictions = predict(X)\n", + "print(predictions)" + ] + }, + { + "cell_type": "markdown", + "id": "bef962e6", + "metadata": { + "editable": true + }, + "source": [ + "Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above." + ] + }, + { + "cell_type": "markdown", + "id": "70caa854", + "metadata": { + "editable": true + }, + "source": [ + "## The Code using Scikit-Learn" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "a1711478", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# import necessary packages\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.neural_network import MLPClassifier\n", + "from sklearn.metrics import accuracy_score\n", + "import seaborn as sns\n", + "\n", + "# ensure the same random numbers appear every time\n", + "np.random.seed(0)\n", + "\n", + "# Design matrix\n", + "X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)\n", + "\n", + "# The XOR gate\n", + "yXOR = np.array( [ 0, 1 ,1, 0])\n", + "# The OR gate\n", + "yOR = np.array( [ 0, 1 ,1, 1])\n", + "# The AND gate\n", + "yAND = np.array( [ 0, 0 ,0, 1])\n", + "\n", + "# Defining the neural network\n", + "n_inputs, n_features = X.shape\n", + "n_hidden_neurons = 2\n", + "n_categories = 2\n", + "n_features = 2\n", + "\n", + "eta_vals = np.logspace(-5, 1, 7)\n", + "lmbd_vals = np.logspace(-5, 1, 7)\n", + "# store models for later use\n", + "DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)\n", + "epochs = 100\n", + "\n", + "for i, eta in enumerate(eta_vals):\n", + " for j, lmbd in enumerate(lmbd_vals):\n", + " dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',\n", + " alpha=lmbd, learning_rate_init=eta, max_iter=epochs)\n", + " dnn.fit(X, yXOR)\n", + " DNN_scikit[i][j] = dnn\n", + " print(\"Learning rate = \", eta)\n", + " print(\"Lambda = \", lmbd)\n", + " print(\"Accuracy score on data set: \", dnn.score(X, yXOR))\n", + " print()\n", + "\n", + "sns.set()\n", + "test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))\n", + "for i in range(len(eta_vals)):\n", + " for j in range(len(lmbd_vals)):\n", + " dnn = DNN_scikit[i][j]\n", + " test_pred = dnn.predict(X)\n", + " test_accuracy[i][j] = accuracy_score(yXOR, test_pred)\n", + "\n", + "fig, ax = plt.subplots(figsize = (10, 10))\n", + "sns.heatmap(test_accuracy, annot=True, ax=ax, cmap=\"viridis\")\n", + "ax.set_title(\"Test Accuracy\")\n", + "ax.set_ylabel(\"$\\eta$\")\n", + "ax.set_xlabel(\"$\\lambda$\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "cd9d7329", + "metadata": { + "editable": true + }, + "source": [ + "## Building a neural network code\n", + "\n", + "Here we present a flexible object oriented codebase\n", + "for a feed forward neural network, along with a demonstration of how\n", + "to use it. Before we get into the details of the neural network, we\n", + "will first present some implementations of various schedulers, cost\n", + "functions and activation functions that can be used together with the\n", + "neural network." + ] + }, + { + "cell_type": "markdown", + "id": "8b9664dc", + "metadata": { + "editable": true + }, + "source": [ + "### Learning rate methods\n", + "\n", + "The code below shows object oriented implementations of the Constant,\n", + "Momentum, Adagrad, AdagradMomentum, RMS prop and Adam schedulers. All\n", + "of the classes belong to the shared abstract Scheduler class, and\n", + "share the update_change() and reset() methods allowing for any of the\n", + "schedulers to be seamlessly used during the training stage, as will\n", + "later be shown in the fit() method of the neural\n", + "network. Update_change() only has one parameter, the gradient\n", + "($δ^l_ja^{l−1}_k$), and returns the change which will be subtracted\n", + "from the weights. The reset() function takes no parameters, and resets\n", + "the desired variables. For Constant and Momentum, reset does nothing." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "97ba7a9a", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import autograd.numpy as np\n", + "\n", + "class Scheduler:\n", + " \"\"\"\n", + " Abstract class for Schedulers\n", + " \"\"\"\n", + "\n", + " def __init__(self, eta):\n", + " self.eta = eta\n", + "\n", + " # should be overwritten\n", + " def update_change(self, gradient):\n", + " raise NotImplementedError\n", + "\n", + " # overwritten if needed\n", + " def reset(self):\n", + " pass\n", + "\n", + "\n", + "class Constant(Scheduler):\n", + " def __init__(self, eta):\n", + " super().__init__(eta)\n", + "\n", + " def update_change(self, gradient):\n", + " return self.eta * gradient\n", + " \n", + " def reset(self):\n", + " pass\n", + "\n", + "\n", + "class Momentum(Scheduler):\n", + " def __init__(self, eta: float, momentum: float):\n", + " super().__init__(eta)\n", + " self.momentum = momentum\n", + " self.change = 0\n", + "\n", + " def update_change(self, gradient):\n", + " self.change = self.momentum * self.change + self.eta * gradient\n", + " return self.change\n", + "\n", + " def reset(self):\n", + " pass\n", + "\n", + "\n", + "class Adagrad(Scheduler):\n", + " def __init__(self, eta):\n", + " super().__init__(eta)\n", + " self.G_t = None\n", + "\n", + " def update_change(self, gradient):\n", + " delta = 1e-8 # avoid division ny zero\n", + "\n", + " if self.G_t is None:\n", + " self.G_t = np.zeros((gradient.shape[0], gradient.shape[0]))\n", + "\n", + " self.G_t += gradient @ gradient.T\n", + "\n", + " G_t_inverse = 1 / (\n", + " delta + np.sqrt(np.reshape(np.diagonal(self.G_t), (self.G_t.shape[0], 1)))\n", + " )\n", + " return self.eta * gradient * G_t_inverse\n", + "\n", + " def reset(self):\n", + " self.G_t = None\n", + "\n", + "\n", + "class AdagradMomentum(Scheduler):\n", + " def __init__(self, eta, momentum):\n", + " super().__init__(eta)\n", + " self.G_t = None\n", + " self.momentum = momentum\n", + " self.change = 0\n", + "\n", + " def update_change(self, gradient):\n", + " delta = 1e-8 # avoid division ny zero\n", + "\n", + " if self.G_t is None:\n", + " self.G_t = np.zeros((gradient.shape[0], gradient.shape[0]))\n", + "\n", + " self.G_t += gradient @ gradient.T\n", + "\n", + " G_t_inverse = 1 / (\n", + " delta + np.sqrt(np.reshape(np.diagonal(self.G_t), (self.G_t.shape[0], 1)))\n", + " )\n", + " self.change = self.change * self.momentum + self.eta * gradient * G_t_inverse\n", + " return self.change\n", + "\n", + " def reset(self):\n", + " self.G_t = None\n", + "\n", + "\n", + "class RMS_prop(Scheduler):\n", + " def __init__(self, eta, rho):\n", + " super().__init__(eta)\n", + " self.rho = rho\n", + " self.second = 0.0\n", + "\n", + " def update_change(self, gradient):\n", + " delta = 1e-8 # avoid division ny zero\n", + " self.second = self.rho * self.second + (1 - self.rho) * gradient * gradient\n", + " return self.eta * gradient / (np.sqrt(self.second + delta))\n", + "\n", + " def reset(self):\n", + " self.second = 0.0\n", + "\n", + "\n", + "class Adam(Scheduler):\n", + " def __init__(self, eta, rho, rho2):\n", + " super().__init__(eta)\n", + " self.rho = rho\n", + " self.rho2 = rho2\n", + " self.moment = 0\n", + " self.second = 0\n", + " self.n_epochs = 1\n", + "\n", + " def update_change(self, gradient):\n", + " delta = 1e-8 # avoid division ny zero\n", + "\n", + " self.moment = self.rho * self.moment + (1 - self.rho) * gradient\n", + " self.second = self.rho2 * self.second + (1 - self.rho2) * gradient * gradient\n", + "\n", + " moment_corrected = self.moment / (1 - self.rho**self.n_epochs)\n", + " second_corrected = self.second / (1 - self.rho2**self.n_epochs)\n", + "\n", + " return self.eta * moment_corrected / (np.sqrt(second_corrected + delta))\n", + "\n", + " def reset(self):\n", + " self.n_epochs += 1\n", + " self.moment = 0\n", + " self.second = 0" + ] + }, + { + "cell_type": "markdown", + "id": "b1b3d6d1", + "metadata": { + "editable": true + }, + "source": [ + "### Usage of the above learning rate schedulers\n", + "\n", + "To initalize a scheduler, simply create the object and pass in the\n", + "necessary parameters such as the learning rate and the momentum as\n", + "shown below. As the Scheduler class is an abstract class it should not\n", + "called directly, and will raise an error upon usage." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "21b4f017", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "momentum_scheduler = Momentum(eta=1e-3, momentum=0.9)\n", + "adam_scheduler = Adam(eta=1e-3, rho=0.9, rho2=0.999)" + ] + }, + { + "cell_type": "markdown", + "id": "eaf81efe", + "metadata": { + "editable": true + }, + "source": [ + "Here is a small example for how a segment of code using schedulers\n", + "could look. Switching out the schedulers is simple." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "3464a296", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "weights = np.ones((3,3))\n", + "print(f\"Before scheduler:\\n{weights=}\")\n", + "\n", + "epochs = 10\n", + "for e in range(epochs):\n", + " gradient = np.random.rand(3, 3)\n", + " change = adam_scheduler.update_change(gradient)\n", + " weights = weights - change\n", + " adam_scheduler.reset()\n", + "\n", + "print(f\"\\nAfter scheduler:\\n{weights=}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d40c1673", + "metadata": { + "editable": true + }, + "source": [ + "### Cost functions\n", + "\n", + "Here we discuss cost functions that can be used when creating the\n", + "neural network. Every cost function takes the target vector as its\n", + "parameter, and returns a function valued only at $x$ such that it may\n", + "easily be differentiated." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "c230465c", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import autograd.numpy as np\n", + "\n", + "def CostOLS(target):\n", + " \n", + " def func(X):\n", + " return (1.0 / target.shape[0]) * np.sum((target - X) ** 2)\n", + "\n", + " return func\n", + "\n", + "\n", + "def CostLogReg(target):\n", + "\n", + " def func(X):\n", + " \n", + " return -(1.0 / target.shape[0]) * np.sum(\n", + " (target * np.log(X + 10e-10)) + ((1 - target) * np.log(1 - X + 10e-10))\n", + " )\n", + "\n", + " return func\n", + "\n", + "\n", + "def CostCrossEntropy(target):\n", + " \n", + " def func(X):\n", + " return -(1.0 / target.size) * np.sum(target * np.log(X + 10e-10))\n", + "\n", + " return func" + ] + }, + { + "cell_type": "markdown", + "id": "b38deb02", + "metadata": { + "editable": true + }, + "source": [ + "Below we give a short example of how these cost function may be used\n", + "to obtain results if you wish to test them out on your own using\n", + "AutoGrad's automatics differentiation." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "9a17387b", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from autograd import grad\n", + "\n", + "target = np.array([[1, 2, 3]]).T\n", + "a = np.array([[4, 5, 6]]).T\n", + "\n", + "cost_func = CostCrossEntropy\n", + "cost_func_derivative = grad(cost_func(target))\n", + "\n", + "valued_at_a = cost_func_derivative(a)\n", + "print(f\"Derivative of cost function {cost_func.__name__} valued at a:\\n{valued_at_a}\")" + ] + }, + { + "cell_type": "markdown", + "id": "dcdaa478", + "metadata": { + "editable": true + }, + "source": [ + "### Activation functions\n", + "\n", + "Finally, before we look at the neural network, we will look at the\n", + "activation functions which can be specified between the hidden layers\n", + "and as the output function. Each function can be valued for any given\n", + "vector or matrix X, and can be differentiated via derivate()." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "902e7512", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import autograd.numpy as np\n", + "from autograd import elementwise_grad\n", + "\n", + "def identity(X):\n", + " return X\n", + "\n", + "\n", + "def sigmoid(X):\n", + " try:\n", + " return 1.0 / (1 + np.exp(-X))\n", + " except FloatingPointError:\n", + " return np.where(X > np.zeros(X.shape), np.ones(X.shape), np.zeros(X.shape))\n", + "\n", + "\n", + "def softmax(X):\n", + " X = X - np.max(X, axis=-1, keepdims=True)\n", + " delta = 10e-10\n", + " return np.exp(X) / (np.sum(np.exp(X), axis=-1, keepdims=True) + delta)\n", + "\n", + "\n", + "def RELU(X):\n", + " return np.where(X > np.zeros(X.shape), X, np.zeros(X.shape))\n", + "\n", + "\n", + "def LRELU(X):\n", + " delta = 10e-4\n", + " return np.where(X > np.zeros(X.shape), X, delta * X)\n", + "\n", + "\n", + "def derivate(func):\n", + " if func.__name__ == \"RELU\":\n", + "\n", + " def func(X):\n", + " return np.where(X > 0, 1, 0)\n", + "\n", + " return func\n", + "\n", + " elif func.__name__ == \"LRELU\":\n", + "\n", + " def func(X):\n", + " delta = 10e-4\n", + " return np.where(X > 0, 1, delta)\n", + "\n", + " return func\n", + "\n", + " else:\n", + " return elementwise_grad(func)" + ] + }, + { + "cell_type": "markdown", + "id": "77182d43", + "metadata": { + "editable": true + }, + "source": [ + "Below follows a short demonstration of how to use an activation\n", + "function. The derivative of the activation function will be important\n", + "when calculating the output delta term during backpropagation. Note\n", + "that derivate() can also be used for cost functions for a more\n", + "generalized approach." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "2723829d", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "z = np.array([[4, 5, 6]]).T\n", + "print(f\"Input to activation function:\\n{z}\")\n", + "\n", + "act_func = sigmoid\n", + "a = act_func(z)\n", + "print(f\"\\nOutput from {act_func.__name__} activation function:\\n{a}\")\n", + "\n", + "act_func_derivative = derivate(act_func)\n", + "valued_at_z = act_func_derivative(a)\n", + "print(f\"\\nDerivative of {act_func.__name__} activation function valued at z:\\n{valued_at_z}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cc6f43b6", + "metadata": { + "editable": true + }, + "source": [ + "### The Neural Network\n", + "\n", + "Now that we have gotten a good understanding of the implementation of\n", + "some important components, we can take a look at an object oriented\n", + "implementation of a feed forward neural network. The feed forward\n", + "neural network has been implemented as a class named FFNN, which can\n", + "be initiated as a regressor or classifier dependant on the choice of\n", + "cost function. The FFNN can have any number of input nodes, hidden\n", + "layers with any amount of hidden nodes, and any amount of output nodes\n", + "meaning it can perform multiclass classification as well as binary\n", + "classification and regression problems. Although there is a lot of\n", + "code present, it makes for an easy to use and generalizeable interface\n", + "for creating many types of neural networks as will be demonstrated\n", + "below." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "e9a07867", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import math\n", + "import autograd.numpy as np\n", + "import sys\n", + "import warnings\n", + "from autograd import grad, elementwise_grad\n", + "from random import random, seed\n", + "from copy import deepcopy, copy\n", + "from typing import Tuple, Callable\n", + "from sklearn.utils import resample\n", + "\n", + "warnings.simplefilter(\"error\")\n", + "\n", + "\n", + "class FFNN:\n", + " \"\"\"\n", + " Description:\n", + " ------------\n", + " Feed Forward Neural Network with interface enabling flexible design of a\n", + " nerual networks architecture and the specification of activation function\n", + " in the hidden layers and output layer respectively. This model can be used\n", + " for both regression and classification problems, depending on the output function.\n", + "\n", + " Attributes:\n", + " ------------\n", + " I dimensions (tuple[int]): A list of positive integers, which specifies the\n", + " number of nodes in each of the networks layers. The first integer in the array\n", + " defines the number of nodes in the input layer, the second integer defines number\n", + " of nodes in the first hidden layer and so on until the last number, which\n", + " specifies the number of nodes in the output layer.\n", + " II hidden_func (Callable): The activation function for the hidden layers\n", + " III output_func (Callable): The activation function for the output layer\n", + " IV cost_func (Callable): Our cost function\n", + " V seed (int): Sets random seed, makes results reproducible\n", + " \"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " dimensions: tuple[int],\n", + " hidden_func: Callable = sigmoid,\n", + " output_func: Callable = lambda x: x,\n", + " cost_func: Callable = CostOLS,\n", + " seed: int = None,\n", + " ):\n", + " self.dimensions = dimensions\n", + " self.hidden_func = hidden_func\n", + " self.output_func = output_func\n", + " self.cost_func = cost_func\n", + " self.seed = seed\n", + " self.weights = list()\n", + " self.schedulers_weight = list()\n", + " self.schedulers_bias = list()\n", + " self.a_matrices = list()\n", + " self.z_matrices = list()\n", + " self.classification = None\n", + "\n", + " self.reset_weights()\n", + " self._set_classification()\n", + "\n", + " def fit(\n", + " self,\n", + " X: np.ndarray,\n", + " t: np.ndarray,\n", + " scheduler: Scheduler,\n", + " batches: int = 1,\n", + " epochs: int = 100,\n", + " lam: float = 0,\n", + " X_val: np.ndarray = None,\n", + " t_val: np.ndarray = None,\n", + " ):\n", + " \"\"\"\n", + " Description:\n", + " ------------\n", + " This function performs the training the neural network by performing the feedforward and backpropagation\n", + " algorithm to update the networks weights.\n", + "\n", + " Parameters:\n", + " ------------\n", + " I X (np.ndarray) : training data\n", + " II t (np.ndarray) : target data\n", + " III scheduler (Scheduler) : specified scheduler (algorithm for optimization of gradient descent)\n", + " IV scheduler_args (list[int]) : list of all arguments necessary for scheduler\n", + "\n", + " Optional Parameters:\n", + " ------------\n", + " V batches (int) : number of batches the datasets are split into, default equal to 1\n", + " VI epochs (int) : number of iterations used to train the network, default equal to 100\n", + " VII lam (float) : regularization hyperparameter lambda\n", + " VIII X_val (np.ndarray) : validation set\n", + " IX t_val (np.ndarray) : validation target set\n", + "\n", + " Returns:\n", + " ------------\n", + " I scores (dict) : A dictionary containing the performance metrics of the model.\n", + " The number of the metrics depends on the parameters passed to the fit-function.\n", + "\n", + " \"\"\"\n", + "\n", + " # setup \n", + " if self.seed is not None:\n", + " np.random.seed(self.seed)\n", + "\n", + " val_set = False\n", + " if X_val is not None and t_val is not None:\n", + " val_set = True\n", + "\n", + " # creating arrays for score metrics\n", + " train_errors = np.empty(epochs)\n", + " train_errors.fill(np.nan)\n", + " val_errors = np.empty(epochs)\n", + " val_errors.fill(np.nan)\n", + "\n", + " train_accs = np.empty(epochs)\n", + " train_accs.fill(np.nan)\n", + " val_accs = np.empty(epochs)\n", + " val_accs.fill(np.nan)\n", + "\n", + " self.schedulers_weight = list()\n", + " self.schedulers_bias = list()\n", + "\n", + " batch_size = X.shape[0] // batches\n", + "\n", + " X, t = resample(X, t)\n", + "\n", + " # this function returns a function valued only at X\n", + " cost_function_train = self.cost_func(t)\n", + " if val_set:\n", + " cost_function_val = self.cost_func(t_val)\n", + "\n", + " # create schedulers for each weight matrix\n", + " for i in range(len(self.weights)):\n", + " self.schedulers_weight.append(copy(scheduler))\n", + " self.schedulers_bias.append(copy(scheduler))\n", + "\n", + " print(f\"{scheduler.__class__.__name__}: Eta={scheduler.eta}, Lambda={lam}\")\n", + "\n", + " try:\n", + " for e in range(epochs):\n", + " for i in range(batches):\n", + " # allows for minibatch gradient descent\n", + " if i == batches - 1:\n", + " # If the for loop has reached the last batch, take all thats left\n", + " X_batch = X[i * batch_size :, :]\n", + " t_batch = t[i * batch_size :, :]\n", + " else:\n", + " X_batch = X[i * batch_size : (i + 1) * batch_size, :]\n", + " t_batch = t[i * batch_size : (i + 1) * batch_size, :]\n", + "\n", + " self._feedforward(X_batch)\n", + " self._backpropagate(X_batch, t_batch, lam)\n", + "\n", + " # reset schedulers for each epoch (some schedulers pass in this call)\n", + " for scheduler in self.schedulers_weight:\n", + " scheduler.reset()\n", + "\n", + " for scheduler in self.schedulers_bias:\n", + " scheduler.reset()\n", + "\n", + " # computing performance metrics\n", + " pred_train = self.predict(X)\n", + " train_error = cost_function_train(pred_train)\n", + "\n", + " train_errors[e] = train_error\n", + " if val_set:\n", + " \n", + " pred_val = self.predict(X_val)\n", + " val_error = cost_function_val(pred_val)\n", + " val_errors[e] = val_error\n", + "\n", + " if self.classification:\n", + " train_acc = self._accuracy(self.predict(X), t)\n", + " train_accs[e] = train_acc\n", + " if val_set:\n", + " val_acc = self._accuracy(pred_val, t_val)\n", + " val_accs[e] = val_acc\n", + "\n", + " # printing progress bar\n", + " progression = e / epochs\n", + " print_length = self._progress_bar(\n", + " progression,\n", + " train_error=train_errors[e],\n", + " train_acc=train_accs[e],\n", + " val_error=val_errors[e],\n", + " val_acc=val_accs[e],\n", + " )\n", + " except KeyboardInterrupt:\n", + " # allows for stopping training at any point and seeing the result\n", + " pass\n", + "\n", + " # visualization of training progression (similiar to tensorflow progression bar)\n", + " sys.stdout.write(\"\\r\" + \" \" * print_length)\n", + " sys.stdout.flush()\n", + " self._progress_bar(\n", + " 1,\n", + " train_error=train_errors[e],\n", + " train_acc=train_accs[e],\n", + " val_error=val_errors[e],\n", + " val_acc=val_accs[e],\n", + " )\n", + " sys.stdout.write(\"\")\n", + "\n", + " # return performance metrics for the entire run\n", + " scores = dict()\n", + "\n", + " scores[\"train_errors\"] = train_errors\n", + "\n", + " if val_set:\n", + " scores[\"val_errors\"] = val_errors\n", + "\n", + " if self.classification:\n", + " scores[\"train_accs\"] = train_accs\n", + "\n", + " if val_set:\n", + " scores[\"val_accs\"] = val_accs\n", + "\n", + " return scores\n", + "\n", + " def predict(self, X: np.ndarray, *, threshold=0.5):\n", + " \"\"\"\n", + " Description:\n", + " ------------\n", + " Performs prediction after training of the network has been finished.\n", + "\n", + " Parameters:\n", + " ------------\n", + " I X (np.ndarray): The design matrix, with n rows of p features each\n", + "\n", + " Optional Parameters:\n", + " ------------\n", + " II threshold (float) : sets minimal value for a prediction to be predicted as the positive class\n", + " in classification problems\n", + "\n", + " Returns:\n", + " ------------\n", + " I z (np.ndarray): A prediction vector (row) for each row in our design matrix\n", + " This vector is thresholded if regression=False, meaning that classification results\n", + " in a vector of 1s and 0s, while regressions in an array of decimal numbers\n", + "\n", + " \"\"\"\n", + "\n", + " predict = self._feedforward(X)\n", + "\n", + " if self.classification:\n", + " return np.where(predict > threshold, 1, 0)\n", + " else:\n", + " return predict\n", + "\n", + " def reset_weights(self):\n", + " \"\"\"\n", + " Description:\n", + " ------------\n", + " Resets/Reinitializes the weights in order to train the network for a new problem.\n", + "\n", + " \"\"\"\n", + " if self.seed is not None:\n", + " np.random.seed(self.seed)\n", + "\n", + " self.weights = list()\n", + " for i in range(len(self.dimensions) - 1):\n", + " weight_array = np.random.randn(\n", + " self.dimensions[i] + 1, self.dimensions[i + 1]\n", + " )\n", + " weight_array[0, :] = np.random.randn(self.dimensions[i + 1]) * 0.01\n", + "\n", + " self.weights.append(weight_array)\n", + "\n", + " def _feedforward(self, X: np.ndarray):\n", + " \"\"\"\n", + " Description:\n", + " ------------\n", + " Calculates the activation of each layer starting at the input and ending at the output.\n", + " Each following activation is calculated from a weighted sum of each of the preceeding\n", + " activations (except in the case of the input layer).\n", + "\n", + " Parameters:\n", + " ------------\n", + " I X (np.ndarray): The design matrix, with n rows of p features each\n", + "\n", + " Returns:\n", + " ------------\n", + " I z (np.ndarray): A prediction vector (row) for each row in our design matrix\n", + " \"\"\"\n", + "\n", + " # reset matrices\n", + " self.a_matrices = list()\n", + " self.z_matrices = list()\n", + "\n", + " # if X is just a vector, make it into a matrix\n", + " if len(X.shape) == 1:\n", + " X = X.reshape((1, X.shape[0]))\n", + "\n", + " # Add a coloumn of zeros as the first coloumn of the design matrix, in order\n", + " # to add bias to our data\n", + " bias = np.ones((X.shape[0], 1)) * 0.01\n", + " X = np.hstack([bias, X])\n", + "\n", + " # a^0, the nodes in the input layer (one a^0 for each row in X - where the\n", + " # exponent indicates layer number).\n", + " a = X\n", + " self.a_matrices.append(a)\n", + " self.z_matrices.append(a)\n", + "\n", + " # The feed forward algorithm\n", + " for i in range(len(self.weights)):\n", + " if i < len(self.weights) - 1:\n", + " z = a @ self.weights[i]\n", + " self.z_matrices.append(z)\n", + " a = self.hidden_func(z)\n", + " # bias column again added to the data here\n", + " bias = np.ones((a.shape[0], 1)) * 0.01\n", + " a = np.hstack([bias, a])\n", + " self.a_matrices.append(a)\n", + " else:\n", + " try:\n", + " # a^L, the nodes in our output layers\n", + " z = a @ self.weights[i]\n", + " a = self.output_func(z)\n", + " self.a_matrices.append(a)\n", + " self.z_matrices.append(z)\n", + " except Exception as OverflowError:\n", + " print(\n", + " \"OverflowError in fit() in FFNN\\nHOW TO DEBUG ERROR: Consider lowering your learning rate or scheduler specific parameters such as momentum, or check if your input values need scaling\"\n", + " )\n", + "\n", + " # this will be a^L\n", + " return a\n", + "\n", + " def _backpropagate(self, X, t, lam):\n", + " \"\"\"\n", + " Description:\n", + " ------------\n", + " Performs the backpropagation algorithm. In other words, this method\n", + " calculates the gradient of all the layers starting at the\n", + " output layer, and moving from right to left accumulates the gradient until\n", + " the input layer is reached. Each layers respective weights are updated while\n", + " the algorithm propagates backwards from the output layer (auto-differentation in reverse mode).\n", + "\n", + " Parameters:\n", + " ------------\n", + " I X (np.ndarray): The design matrix, with n rows of p features each.\n", + " II t (np.ndarray): The target vector, with n rows of p targets.\n", + " III lam (float32): regularization parameter used to punish the weights in case of overfitting\n", + "\n", + " Returns:\n", + " ------------\n", + " No return value.\n", + "\n", + " \"\"\"\n", + " out_derivative = derivate(self.output_func)\n", + " hidden_derivative = derivate(self.hidden_func)\n", + "\n", + " for i in range(len(self.weights) - 1, -1, -1):\n", + " # delta terms for output\n", + " if i == len(self.weights) - 1:\n", + " # for multi-class classification\n", + " if (\n", + " self.output_func.__name__ == \"softmax\"\n", + " ):\n", + " delta_matrix = self.a_matrices[i + 1] - t\n", + " # for single class classification\n", + " else:\n", + " cost_func_derivative = grad(self.cost_func(t))\n", + " delta_matrix = out_derivative(\n", + " self.z_matrices[i + 1]\n", + " ) * cost_func_derivative(self.a_matrices[i + 1])\n", + "\n", + " # delta terms for hidden layer\n", + " else:\n", + " delta_matrix = (\n", + " self.weights[i + 1][1:, :] @ delta_matrix.T\n", + " ).T * hidden_derivative(self.z_matrices[i + 1])\n", + "\n", + " # calculate gradient\n", + " gradient_weights = self.a_matrices[i][:, 1:].T @ delta_matrix\n", + " gradient_bias = np.sum(delta_matrix, axis=0).reshape(\n", + " 1, delta_matrix.shape[1]\n", + " )\n", + "\n", + " # regularization term\n", + " gradient_weights += self.weights[i][1:, :] * lam\n", + "\n", + " # use scheduler\n", + " update_matrix = np.vstack(\n", + " [\n", + " self.schedulers_bias[i].update_change(gradient_bias),\n", + " self.schedulers_weight[i].update_change(gradient_weights),\n", + " ]\n", + " )\n", + "\n", + " # update weights and bias\n", + " self.weights[i] -= update_matrix\n", + "\n", + " def _accuracy(self, prediction: np.ndarray, target: np.ndarray):\n", + " \"\"\"\n", + " Description:\n", + " ------------\n", + " Calculates accuracy of given prediction to target\n", + "\n", + " Parameters:\n", + " ------------\n", + " I prediction (np.ndarray): vector of predicitons output network\n", + " (1s and 0s in case of classification, and real numbers in case of regression)\n", + " II target (np.ndarray): vector of true values (What the network ideally should predict)\n", + "\n", + " Returns:\n", + " ------------\n", + " A floating point number representing the percentage of correctly classified instances.\n", + " \"\"\"\n", + " assert prediction.size == target.size\n", + " return np.average((target == prediction))\n", + " def _set_classification(self):\n", + " \"\"\"\n", + " Description:\n", + " ------------\n", + " Decides if FFNN acts as classifier (True) og regressor (False),\n", + " sets self.classification during init()\n", + " \"\"\"\n", + " self.classification = False\n", + " if (\n", + " self.cost_func.__name__ == \"CostLogReg\"\n", + " or self.cost_func.__name__ == \"CostCrossEntropy\"\n", + " ):\n", + " self.classification = True\n", + "\n", + " def _progress_bar(self, progression, **kwargs):\n", + " \"\"\"\n", + " Description:\n", + " ------------\n", + " Displays progress of training\n", + " \"\"\"\n", + " print_length = 40\n", + " num_equals = int(progression * print_length)\n", + " num_not = print_length - num_equals\n", + " arrow = \">\" if num_equals > 0 else \"\"\n", + " bar = \"[\" + \"=\" * (num_equals - 1) + arrow + \"-\" * num_not + \"]\"\n", + " perc_print = self._format(progression * 100, decimals=5)\n", + " line = f\" {bar} {perc_print}% \"\n", + "\n", + " for key in kwargs:\n", + " if not np.isnan(kwargs[key]):\n", + " value = self._format(kwargs[key], decimals=4)\n", + " line += f\"| {key}: {value} \"\n", + " sys.stdout.write(\"\\r\" + line)\n", + " sys.stdout.flush()\n", + " return len(line)\n", + "\n", + " def _format(self, value, decimals=4):\n", + " \"\"\"\n", + " Description:\n", + " ------------\n", + " Formats decimal numbers for progress bar\n", + " \"\"\"\n", + " if value > 0:\n", + " v = value\n", + " elif value < 0:\n", + " v = -10 * value\n", + " else:\n", + " v = 1\n", + " n = 1 + math.floor(math.log10(v))\n", + " if n >= decimals - 1:\n", + " return str(round(value))\n", + " return f\"{value:.{decimals-n-1}f}\"" + ] + }, + { + "cell_type": "markdown", + "id": "f957b537", + "metadata": { + "editable": true + }, + "source": [ + "Before we make a model, we will quickly generate a dataset we can use\n", + "for our linear regression problem as shown below" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "97018c57", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import autograd.numpy as np\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "def SkrankeFunction(x, y):\n", + " return np.ravel(0 + 1*x + 2*y + 3*x**2 + 4*x*y + 5*y**2)\n", + "\n", + "def create_X(x, y, n):\n", + " if len(x.shape) > 1:\n", + " x = np.ravel(x)\n", + " y = np.ravel(y)\n", + "\n", + " N = len(x)\n", + " l = int((n + 1) * (n + 2) / 2) # Number of elements in beta\n", + " X = np.ones((N, l))\n", + "\n", + " for i in range(1, n + 1):\n", + " q = int((i) * (i + 1) / 2)\n", + " for k in range(i + 1):\n", + " X[:, q + k] = (x ** (i - k)) * (y**k)\n", + "\n", + " return X\n", + "\n", + "step=0.5\n", + "x = np.arange(0, 1, step)\n", + "y = np.arange(0, 1, step)\n", + "x, y = np.meshgrid(x, y)\n", + "target = SkrankeFunction(x, y)\n", + "target = target.reshape(target.shape[0], 1)\n", + "\n", + "poly_degree=3\n", + "X = create_X(x, y, poly_degree)\n", + "\n", + "X_train, X_test, t_train, t_test = train_test_split(X, target)" + ] + }, + { + "cell_type": "markdown", + "id": "c46a864f", + "metadata": { + "editable": true + }, + "source": [ + "Now that we have our dataset ready for the regression, we can create\n", + "our regressor. Note that with the seed parameter, we can make sure our\n", + "results stay the same every time we run the neural network. For\n", + "inititialization, we simply specify the dimensions (we wish the amount\n", + "of input nodes to be equal to the datapoints, and the output to\n", + "predict one value)." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "da5c2906", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "input_nodes = X_train.shape[1]\n", + "output_nodes = 1\n", + "\n", + "linear_regression = FFNN((input_nodes, output_nodes), output_func=identity, cost_func=CostOLS, seed=2023)" + ] + }, + { + "cell_type": "markdown", + "id": "ec8e4fce", + "metadata": { + "editable": true + }, + "source": [ + "We then fit our model with our training data using the scheduler of our choice." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "9e63ffa2", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "linear_regression.reset_weights() # reset weights such that previous runs or reruns don't affect the weights\n", + "\n", + "scheduler = Constant(eta=1e-3)\n", + "scores = linear_regression.fit(X_train, t_train, scheduler)" + ] + }, + { + "cell_type": "markdown", + "id": "39cf3c39", + "metadata": { + "editable": true + }, + "source": [ + "Due to the progress bar we can see the MSE (train_error) throughout\n", + "the FFNN's training. Note that the fit() function has some optional\n", + "parameters with defualt arguments. For example, the regularization\n", + "hyperparameter can be left ignored if not needed, and equally the FFNN\n", + "will by default run for 100 epochs. These can easily be changed, such\n", + "as for example:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "a50c2400", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "linear_regression.reset_weights() # reset weights such that previous runs or reruns don't affect the weights\n", + "\n", + "scores = linear_regression.fit(X_train, t_train, scheduler, lam=1e-4, epochs=1000)" + ] + }, + { + "cell_type": "markdown", + "id": "59e3767e", + "metadata": { + "editable": true + }, + "source": [ + "We see that given more epochs to train on, the regressor reaches a lower MSE.\n", + "\n", + "Let us then switch to a binary classification. We use a binary\n", + "classification dataset, and follow a similar setup to the regression\n", + "case." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "c77d9ebd", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from sklearn.datasets import load_breast_cancer\n", + "from sklearn.preprocessing import MinMaxScaler\n", + "\n", + "wisconsin = load_breast_cancer()\n", + "X = wisconsin.data\n", + "target = wisconsin.target\n", + "target = target.reshape(target.shape[0], 1)\n", + "\n", + "X_train, X_val, t_train, t_val = train_test_split(X, target)\n", + "\n", + "scaler = MinMaxScaler()\n", + "scaler.fit(X_train)\n", + "X_train = scaler.transform(X_train)\n", + "X_val = scaler.transform(X_val)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "7a872564", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "input_nodes = X_train.shape[1]\n", + "output_nodes = 1\n", + "\n", + "logistic_regression = FFNN((input_nodes, output_nodes), output_func=sigmoid, cost_func=CostLogReg, seed=2023)" + ] + }, + { + "cell_type": "markdown", + "id": "cc5ad007", + "metadata": { + "editable": true + }, + "source": [ + "We will now make use of our validation data by passing it into our fit function as a keyword argument" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "fac34043", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "logistic_regression.reset_weights() # reset weights such that previous runs or reruns don't affect the weights\n", + "\n", + "scheduler = Adam(eta=1e-3, rho=0.9, rho2=0.999)\n", + "scores = logistic_regression.fit(X_train, t_train, scheduler, epochs=1000, X_val=X_val, t_val=t_val)" + ] + }, + { + "cell_type": "markdown", + "id": "9a581657", + "metadata": { + "editable": true + }, + "source": [ + "Finally, we will create a neural network with 2 hidden layers with activation functions." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "3421ca31", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "input_nodes = X_train.shape[1]\n", + "hidden_nodes1 = 100\n", + "hidden_nodes2 = 30\n", + "output_nodes = 1\n", + "\n", + "dims = (input_nodes, hidden_nodes1, hidden_nodes2, output_nodes)\n", + "\n", + "neural_network = FFNN(dims, hidden_func=RELU, output_func=sigmoid, cost_func=CostLogReg, seed=2023)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "bb6f3c50", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "neural_network.reset_weights() # reset weights such that previous runs or reruns don't affect the weights\n", + "\n", + "scheduler = Adam(eta=1e-4, rho=0.9, rho2=0.999)\n", + "scores = neural_network.fit(X_train, t_train, scheduler, epochs=1000, X_val=X_val, t_val=t_val)" + ] + }, + { + "cell_type": "markdown", + "id": "6e719f18", + "metadata": { + "editable": true + }, + "source": [ + "### Multiclass classification\n", + "\n", + "Finally, we will demonstrate the use case of multiclass classification\n", + "using our FFNN with the famous MNIST dataset, which contain images of\n", + "digits between the range of 0 to 9." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "bba5ad80", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from sklearn.datasets import load_digits\n", + "\n", + "def onehot(target: np.ndarray):\n", + " onehot = np.zeros((target.size, target.max() + 1))\n", + " onehot[np.arange(target.size), target] = 1\n", + " return onehot\n", + "\n", + "digits = load_digits()\n", + "\n", + "X = digits.data\n", + "target = digits.target\n", + "target = onehot(target)\n", + "\n", + "input_nodes = 64\n", + "hidden_nodes1 = 100\n", + "hidden_nodes2 = 30\n", + "output_nodes = 10\n", + "\n", + "dims = (input_nodes, hidden_nodes1, hidden_nodes2, output_nodes)\n", + "\n", + "multiclass = FFNN(dims, hidden_func=LRELU, output_func=softmax, cost_func=CostCrossEntropy)\n", + "\n", + "multiclass.reset_weights() # reset weights such that previous runs or reruns don't affect the weights\n", + "\n", + "scheduler = Adam(eta=1e-4, rho=0.9, rho2=0.999)\n", + "scores = multiclass.fit(X, target, scheduler, epochs=1000)" + ] } ], "metadata": {}, diff --git a/doc/LectureNotes/_build/html/_sources/week43.ipynb b/doc/LectureNotes/_build/html/_sources/week43.ipynb index db8682062..e80f0e0d1 100644 --- a/doc/LectureNotes/_build/html/_sources/week43.ipynb +++ b/doc/LectureNotes/_build/html/_sources/week43.ipynb @@ -3,9 +3,7 @@ { "cell_type": "markdown", "id": "944a1ec4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", @@ -15,9 +13,7 @@ { "cell_type": "markdown", "id": "571ef4ca", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "# Week 43: Deep Learning: Constructing a Neural Network code and solving differential equations\n", "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and Facility for Rare Isotope Beams, Michigan State University\n", @@ -30,9 +26,7 @@ { "cell_type": "markdown", "id": "c53e0e9f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Plans for week 43\n", "\n", @@ -74,9 +68,7 @@ { "cell_type": "markdown", "id": "02ed9bb6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Using Automatic differentiation\n", "a\n", @@ -88,9 +80,7 @@ { "cell_type": "markdown", "id": "05cfb0c9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Back propagation and automatic differentiation\n", "\n", @@ -105,9 +95,7 @@ { "cell_type": "markdown", "id": "21caf391", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Material for exercises week 43 and week 44" ] @@ -115,9 +103,7 @@ { "cell_type": "markdown", "id": "3c9b1e04", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Writing our first neural network code, testing it for the OR and XOR gates\n", "\n", @@ -152,9 +138,7 @@ { "cell_type": "markdown", "id": "4dabb457", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The AND and XOR Gates\n", "\n", @@ -190,9 +174,7 @@ { "cell_type": "markdown", "id": "a78cff46", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Representing the Data Sets\n", "\n", @@ -202,9 +184,7 @@ { "cell_type": "markdown", "id": "e4c2f9d7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}=\\begin{bmatrix} 0 & 0 \\\\\n", @@ -217,9 +197,7 @@ { "cell_type": "markdown", "id": "103736e3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "while the vector of outputs is $\\boldsymbol{y}^T=[0,1,1,0]$ for the XOR gate, $\\boldsymbol{y}^T=[0,0,0,1]$ for the AND gate and $\\boldsymbol{y}^T=[0,1,1,1]$ for the OR gate." ] @@ -227,9 +205,7 @@ { "cell_type": "markdown", "id": "a5e5d384", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Setting up the Neural Network\n", "\n", @@ -240,11 +216,20 @@ "cell_type": "code", "execution_count": 1, "id": "09d0e5dd", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[0.80625657 0.36420967]\n", + " [0.90297441 0.30170017]\n", + " [0.89823921 0.28566769]\n", + " [0.93420126 0.25920793]]\n", + "[0 0 0 0]\n" + ] + } + ], "source": [ "%matplotlib inline\n", "\n", @@ -318,9 +303,7 @@ { "cell_type": "markdown", "id": "d9f554db", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above." ] @@ -328,9 +311,7 @@ { "cell_type": "markdown", "id": "ab802e6b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The Code using Scikit-Learn" ] @@ -339,11 +320,253 @@ "cell_type": "code", "execution_count": 2, "id": "9231d583", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Learning rate = 1e-05\n", + "Lambda = 1e-05\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 1e-05\n", + "Lambda = 0.0001\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 1e-05\n", + "Lambda = 0.001\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 1e-05\n", + "Lambda = 0.01\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 1e-05\n", + "Lambda = 0.1\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 1e-05\n", + "Lambda = 1.0\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 1e-05\n", + "Lambda = 10.0\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.0001\n", + "Lambda = 1e-05\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.0001\n", + "Lambda = 0.0001\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.0001\n", + "Lambda = 0.001\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.0001\n", + "Lambda = 0.01\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.0001\n", + "Lambda = 0.1\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.0001\n", + "Lambda = 1.0\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.0001\n", + "Lambda = 10.0\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.001\n", + "Lambda = 1e-05\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.001\n", + "Lambda = 0.0001\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.001\n", + "Lambda = 0.001\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.001\n", + "Lambda = 0.01\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.001\n", + "Lambda = 0.1\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.001\n", + "Lambda = 1.0\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.001\n", + "Lambda = 10.0\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.01\n", + "Lambda = 1e-05\n", + "Accuracy score on data set: 0.25\n", + "\n", + "Learning rate = 0.01\n", + "Lambda = 0.0001\n", + "Accuracy score on data set: 0.75\n", + "\n", + "Learning rate = 0.01\n", + "Lambda = 0.001\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.01\n", + "Lambda = 0.01\n", + "Accuracy score on data set: 0.75\n", + "\n", + "Learning rate = 0.01\n", + "Lambda = 0.1\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.01\n", + "Lambda = 1.0\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.01\n", + "Lambda = 10.0\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.1\n", + "Lambda = 1e-05\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.1\n", + "Lambda = 0.0001\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.1\n", + "Lambda = 0.001\n", + "Accuracy score on data set: 1.0\n", + "\n", + "Learning rate = 0.1\n", + "Lambda = 0.01\n", + "Accuracy score on data set: 1.0\n", + "\n", + "Learning rate = 0.1\n", + "Lambda = 0.1\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.1\n", + "Lambda = 1.0\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 0.1\n", + "Lambda = 10.0\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 1.0\n", + "Lambda = 1e-05\n", + "Accuracy score on data set: 0.75\n", + "\n", + "Learning rate = 1.0\n", + "Lambda = 0.0001\n", + "Accuracy score on data set: 0.75\n", + "\n", + "Learning rate = 1.0\n", + "Lambda = 0.001\n", + "Accuracy score on data set: 0.75\n", + "\n", + "Learning rate = 1.0\n", + "Lambda = 0.01\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 1.0\n", + "Lambda = 0.1\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 1.0\n", + "Lambda = 1.0\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 1.0\n", + "Lambda = 10.0\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 10.0\n", + "Lambda = 1e-05\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 10.0\n", + "Lambda = 0.0001\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 10.0\n", + "Lambda = 0.001\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 10.0\n", + "Lambda = 0.01\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 10.0\n", + "Lambda = 0.1\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 10.0\n", + "Lambda = 1.0\n", + "Accuracy score on data set: 0.5\n", + "\n", + "Learning rate = 10.0\n", + "Lambda = 10.0\n", + "Accuracy score on data set: 0.5\n", + "\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAyAAAANbCAYAAAC6lftqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAABrR0lEQVR4nO3dd3xUddbH8e9QJgMpQISQSOgEYpQSDEVpIqDriuIKPsqKC4tEmrCCCzYUUGwLigKCq1J0kQWlWigi7q4U6ShCCE1aME0gCSE9mecPJLvjBBhg7r2Z5PN+XvN6mN/cO/fk7BA5c8691+Z0Op0CAAAAABNUsDoAAAAAAOUHBQgAAAAA01CAAAAAADANBQgAAAAA01CAAAAAADANBQgAAAAA01CAAAAAADANBQgAAAAA01CAAICBuNcrAACuKlkdAACY6emnn9ayZcsuuU2dOnX0zTffXPOxPv30Ux0+fFhPP/30ZbddvHixnnvuOXXq1EkffPDBNR8bAIDSyubk6zkA5cjx48d1+vTp4uczZ85UXFycZsyYUbxmt9sVFRV1zce6/fbb1bZtW7322muX3bZv377KzMzUoUOH9NVXX6lu3brXfHwAAEojOiAAypV69eqpXr16xc+Dg4Nlt9vVqlUry2I6cuSIdu7cqffee09jxozRJ598oieffNKyeAAAMBLngABACQ4cOKDBgwerdevWat26tYYPH64TJ064bPOPf/xDv/vd79S8eXN16tRJEyZMUGZmpqTz3Y+TJ09q2bJlatasmRISEi56rCVLligwMFC33HKLfve732nJkiXKy8tz227Pnj0aNGiQbr75ZrVv316jRo1SYmJi8eunTp3Ss88+q1tvvVXR0dF6+OGHtWPHjuLXmzVrpunTp7u85/Tp09WsWbPi508//bT69++v8ePHKyYmRn/4wx9UUFCg06dPa+LEieratatuuukmtW3bVsOHD3f7ub788kvdf//9atmypW677TZNnjxZeXl5OnjwoJo1a6ZFixa5bJ+cnKwbbrjhsmNxAICygwIEAH7jyJEjeuihh3Tq1Cm99tprevnll3XixAn17dtXp06dknT+H9qvv/66Hn74Yc2ePVvDhw/XihUrNGnSJEnSjBkzVKtWLXXp0kWLFi1SSEhIiccqLCzUihUr9Pvf/152u13333+/Tp06pa+//tplu/j4ePXt21fZ2dl67bXX9OKLLyouLk4DBw5Ufn6+srKy9NBDD2nTpk168sknNWPGDPn7+2vQoEE6fPjwFf3827dv17FjxzR9+nQNHz5cFStW1ODBg7Vx40Y9+eSTmj17toYNG6ZNmzbphRdeKN5v4cKFGj16tG644QbNmDFDgwcP1oIFCzRhwgRFRESoZcuWWrFihcuxVqxYIYfDoTvvvPOKYgQA+C5GsADgN2bMmCGHw6F58+YpICBAknTLLbeoe/fu+uCDD/TUU09py5YtqlOnjh5++GFVqFBBbdu2VdWqVXXmzBlJUlRUlOx2u4KDgy853vXtt98qJSVFvXv3liS1atVKTZo00T//+U/9/ve/L95u5syZqlatmubMmSM/Pz9JUmhoqJ544gnt379fP/zwg06cOKHly5crMjJSkhQTE6P77rtP27ZtU+PGjT3++QsKCjRx4kTVr19f0vkuRZUqVfTUU08pJiZGktSuXTslJCRo4cKFkqSioiJNnz5dPXr00Msvv1z8Xrm5uVq2bJny8vLUu3dvvfDCCzpx4kTxOS7Lly/XXXfdpapVq3ocHwDAt9EBAYDf2Lx5s9q1ayeHw6GCggIVFBQoICBAMTEx2rRpkySpffv2Onr0qO6///7iE9nvuece9e/f/4qOtWTJEtWvX18NGzZURkaGMjIydNddd2nr1q0unYsdO3aoc+fOxcWHJLVo0ULffPONbrrpJm3fvl3h4eHFxYck+fn5adWqVXrooYeuKCaHw+Fynkzt2rX10UcfKSYmRj///LO+++47zZ8/Xzt37lR+fr6k812jX375Rd27d3d5rwEDBmjFihWy2+26++67VaVKleIuyO7du3X48GHdf//9VxQfAMC30QEBgN9IS0vTypUrtXLlSrfXgoODJUm///3vVVRUpAULFmjGjBl6++23VadOHT355JO6++67PTrO6dOn9e9//1v5+flq06aN2+uLFi3Ss88+WxzTddddd8mYL/X6lbjuuutks9lc1j777DO9+eabSkxMVPXq1RUZGSmHw+Fy/Av7XkxAQIB+97vf6bPPPtPjjz+uZcuWqX79+sVdFQBA+UABAgC/ERgYqFtvvVV//vOf3V6rVOm/vzZ79uypnj176uzZs9qwYYPef/99jRkzRjExMapdu/Zlj7NixQrl5+drxowZCgoKcnntnXfe0fLlyzV69Gg5HA4FBga6XD74gv/85z+KjIxUYGBgiSe679q1SwEBAYqIiJB0/pyT/5WVlXXZOLdv366nnnpK/fr106OPPqrQ0FBJ0t/+9rfik9wvxP/bGNPS0rR37161atVK/v7+6t27t5YtW6bdu3drzZo1euSRRy57fABA2cIIFgD8Rtu2bXXo0CHdcMMNat68uZo3b66bbrpJ8+bN09q1ayVJTzzxhB5//HFJ5wuWu+66S8OGDVNhYaFSUlIkSRUqXPpX7NKlS9WqVSv16NFD7dq1c3n07dtX6enpWrVqlaTz53OsX7/e5epY+/fv12OPPaYff/xRMTExOnHihPbv31/8el5enkaMGKFPPvlE0vkORFJSkksMO3fuvGw+du3apaKiIo0cObK4+CgsLCweRysqKlKjRo1Uo0YNrVu3zmXfzz//XLGxscrNzZUktWnTRg0aNNDkyZN15swZ3XfffZc9PgCgbKEAAYDfGDZsmI4fP67Bgwfr66+/1vr16zVixAh9+eWXxedYtG/fXmvXrtXrr7+u7777TmvWrNHbb7+tBg0aFG8TFBSkuLg4bd26VTk5OS7H2L17tw4cOHDRca1u3bqpWrVqxSd5Dxs2TGfOnFFsbKy++eYbrV69Wk888YRuvPFGde7cWffff7/q1q2roUOHasWKFVq/fr1GjhypnJyc4i7Dbbfdpi+//FILFizQd999p7Fjx+rYsWOXzUeLFi0kSS+++KI2b96sr776Sn/+858VHx8v6XwXpWLFihoxYoTWrFmjCRMmaOPGjfr444/11ltvqW/fvsWja5LUu3dvbd26VbfccovCwsKu5H8aAEAZQAECAL8RGRmpjz/+WDabTWPHjtXIkSOVmpqqd955R3fccYck6aGHHtK4ceP07bffasiQIXrhhRfUuHFjzZkzR5UrV5YkDRw4UL/88oseffRR7dmzx+UYS5YsUcWKFV2udPW/7Ha77rrrLn3//ffat2+foqKi9I9//ENFRUUaNWqUXnzxRbVq1Urvv/++7Ha7AgICNH/+fEVHR+vll1/WX/7yF+Xm5uof//hH8QnlzzzzjG6//XZNnjxZI0eOVJUqVTy64WG7du30wgsvaNeuXYqNjdWrr76q66+/vvju8RfGsB5++GG99tpr2r59uwYPHqw5c+Zo4MCBevrpp13e77bbbpMkTj4HgHLK5nQ6nVYHAQAoP95//3198MEHWr9+vex2u9XhAABMxknoAABTLFu2TAcOHNCCBQv02GOPUXwAQDlFAQIAMEV8fLwWLlyo7t27KzY21upwAAAWYQQLAAAAgGbOnKnvvvtO//jHPy66zZkzZzRp0iR9++23kqTf/e53euaZZ1S1alWPj8NJ6AAAAEA5N2/ePE2bNu2y240cOVInTpwo3n7jxo2aOHHiFR2LESwAAACgnEpOTtZzzz2nHTt2qGHDhpfcdteuXdq6datWrlypxo0bSzp/ifZBgwZp9OjRHt2EV6IDAgAAAJRbe/fuVbVq1fTZZ5+pZcuWl9x2+/btqlWrVnHxIZ2/ea/NZiu+JLsn6IAAAAAAPqxbt26XfH3dunUXfe3222/X7bff7tFxkpOT3W4ga7fbVb16dSUmJnr0HlI5K0BaD51qdQgAAAC4hJ2zRlkdwkUVJTW1OoSLqGvKUbKzs0u8hLqfn59yc3M9fp9yVYAAAAAAZc2lOhze5HA4lJeX57aem5vLVbAAAAAAeFdoaKhSUlJc1vLy8pSWlubxCegSBQgAAADgkaJS+n9madOmjZKSknTs2LHitS1btkiSWrdu7fH7UIAAAAAAcFNYWKjU1FTl5ORIklq2bKnWrVtr1KhR2r17tzZv3qzx48frvvvuowMCAAAA4NokJiaqY8eOWrlypSTJZrNpxowZCg8PV//+/fXEE0+oc+fOmjBhwhW9r83pdDoNiLdU4ipYAAAApVtpvgpWbmIjq0MokV/YT1aHcEXogAAAAAAwDQUIAAAAANNwHxAAAADAA0UqN2cuGIoOCAAAAADTUIAAAAAAMA0jWAAAAIAHzLzpX1lGBwQAAACAaShAAAAAAJiGESwAAADAA4Xl5/7dhqIDAgAAAMA0FCAAAAAATMMIFgAAAOABbkToHXRAAAAAAJiGAgQAAACAaRjBAgAAADxQyAiWV9ABAQAAAGAaChAAAAAApmEECwAAAPAAV8HyDjogAAAAAExDAQIAAADANIxgAQAAAB4odDKC5Q10QAAAAACYhgIEAAAAgGkYwQIAAAA8UGR1AGUEHRAAAAAApqEAAQAAAGAaRrAAAAAADxRyI0KvoAMCAAAAwDQUIAAAAABMwwgWAAAA4IFCJrC8gg4IAAAAANNQgAAAAAAwDSNYAAAAgAe4EaF30AEBAAAAYBoKEAAAAACmYQQLAAAA8EChbFaHUCbQAQEAAABgGgoQAAAAAKZhBAsAAADwQBE3IvQKOiAAAAAATEMBAgAAAMA0jGBZ5Nao+hp2bwc1DAtW2tlsLV6/W3PXbLvo9g1q19DSCQPc1o8knVbviR8aGKnvIsfGIr/GI8fGI8fGIr/GI8fm4ipY3kEBYoEWjcI0dWgvfbXjgGZ+tlGtmtTR8Hs7qILNptmrt5a4T9O6tSRJsW9+qryCguL1nLyCErcv78ixsciv8cix8cixsciv8cgxfBUFiAUG391e+xNS9fy81ZKkTXHHVKlCBQ24s43mr9uh3PxCt32ahYfo5C/p2nEwwexwfRI5Nhb5NR45Nh45Nhb5NR45hq/iHBCTVa5UUTdHhOubXYdc1r/edVD+Druim9Qpcb9m4bW0PyHVjBB9Hjk2Fvk1Hjk2Hjk2Fvk1Hjm2RqFspfLha+iAmCy8ZjXZK1fS8ZQzLusnUtMkSfVDamjzvuNu+zUNr6UjSac0b8yDalY3RGezcvX55jjN+myTCoqKzAjdZ5BjY5Ff45Fj45FjY5Ff45Fj+DLLC5CCggJ99dVX2r59u37++Wfl5eWpSpUqCg0NVUxMjHr06KFKlSwP02sCq/hJkjJz8lzWs3597l/F7rZPcGBV1azmL6fTqWnLNijxTIbaNqunAXfEqHaNAI2bu9r4wH0IOTYW+TUeOTYeOTYW+TUeOYYvs/Rf9sePH1dsbKySk5MVFRWlkJAQVatWTbm5udq3b5+WLFmi6dOn64MPPtD1119vZaheY7P92iZzlnwnm5K+fDiXk6shby/RseTTSj6TKUnaefCk8gsKNbxXB81etVVHkk4bFbLPIcfGIr/GI8fGI8fGIr/GI8fWKHL63rhTaWTpOSATJ05UeHi41q9frwULFuitt97S66+/rrfeeksLFizQt99+q7CwML344otWhulVZ7NzJUn+DtdvJqr++jwzJ9dtn9z8Qm2NP178y+KC9XuOSJIiwmsaEarPIsfGIr/GI8fGI8fGIr/GI8fwZZYWIDt27NDYsWMVGBhY4utBQUEaM2aMtm27+PWsfU1CapoKCotUN6S6y3rdWuef/5R4ym2f+rVrqHenFm6/ZByVzzew0jJzDInVV5FjY5Ff45Fj45FjY5Ff45Fj+DJLC5CgoCClpKRccpuff/5ZDofDpIiMl1dQqF2HEnR7qyYu692jI5SRlaO9R5Pc9gmpHqDn/thN3aMjXNbviGmqzOxc7TuebGjMvoYcG4v8Go8cG48cG4v8Go8cW8Pqq11xFSwv6NOnj5555hmNHDlS7dq1U1hYmOx2u/Ly8pScnKytW7dqypQp6tOnj5Vhet0Hq7Zq1sjeen3Q3Vrx3V61bBSmP/WI0bTl65WbXyh/h12NwoJ1IjVdaZnZ2nEgQdv2n9DoPp3l8Kuso0mn1al5Qz10W7TeWvqtzma5t1nLO3JsLPJrPHJsPHJsLPJrPHIMX2VzOi9y9pIJnE6n3nnnHc2dO1dZWVlur/v7++vhhx/WX/7yF1WocO3NmtZDp17ze3hL15aNNaTnLapfu4ZS0s/pk39/r/nrdkqSbo4I1/ujH9D4D9fo881xkqQAh12De96i21o2Vs1q/kpITdeCb3Zq2cY9Vv4YpRo5Nhb5NR45Nh45Nhb5NV5ZzPHOWaOsDuGidh6vZ3UIJWpdz/2Sy6WZpQXIBfn5+dq3b5+Sk5OVnZ0th8Oh0NBQRUZGym53v4zc1SpNBQgAAADcleYCZNvxBlaHUKI29Y5aHcIVKRU32KhcubJatGhhdRgAAAAADGbpSegAAAAAypdS0QEBAAAASjtuROgddEAAAAAAmIYCBAAAAIBpGMECAAAAPOCLN/0rjeiAAAAAADANBQgAAAAA0zCCBQAAAHig0Ml3995AFgEAAACYhgIEAAAAgGkYwQIAAAA8UMR3915BFgEAAACYhgIEAAAAgGkYwQIAAAA8wI0IvYMOCAAAAADTUIAAAAAAMA0jWAAAAIAHuBGhd5BFAAAAAKahAAEAAABgGkawAAAAAA8UcRUsr6ADAgAAAMA0FCAAAAAATMMIFgAAAOCBQr679wqyCAAAAMA0FCAAAAAATMMIFgAAAOABbkToHWQRAAAAgGkoQAAAAACYhhEsAAAAwANFfHfvFWQRAAAAgGkoQAAAAACYhhEsAAAAwAOFTpvVIZQJdEAAAAAAmIYCBAAAAIBpKEAAAAAAmIZzQAAAAAAPFPLdvVeQRQAAAKCcKioq0rRp09SpUye1bNlSAwcO1LFjxy66/YkTJzRkyBC1bdtWHTp00KRJk5SdnX1Fx6QAAQAAAMqpmTNnauHChZo0aZIWLVokm82m2NhY5eXluW179uxZ9e3bV+np6frggw/07rvvas+ePRo+fPgVHZMCBAAAAPBAkbNCqXxcrby8PM2ZM0cjRoxQly5dFBkZqalTpyo5OVlr1651237ZsmXKzMzUO++8oxYtWqh58+aaOnWqNm3apO3bt3t8XAoQAAAAoByKj4/XuXPn1L59++K1oKAgRUVFadu2bW7bHzlyRI0aNVJwcHDxWlhYmGrUqKGtW7d6fFxOQgcAAAB8WLdu3S75+rp160pcT0pKknS+iPhfISEhSkxMdNu+Vq1aSk1NVWFhoSpWrChJyszMVHp6uk6dOuVxvHRAAAAAAA8UqkKpfFytCyeP2+12l3U/Pz/l5ua6bX/33XcrPT1dr7zyis6dO6eMjAyNHz9eNputxHNGLoYOCAAAAODDLtbhuByHwyHp/LkgF/4sSbm5uapSpYrb9vXr19f06dP1wgsv6OOPP5bD4dAjjzyim266SQEBAR4flwIEAAAAKIcujF6lpKSoXr16xespKSmKjIwscZ8uXbroP//5j1JTUxUYGCiHw6Fbb71V999/v8fHZQQLAAAA8ECh01YqH1crMjJSAQEB2rJlS/FaRkaG4uLiFBMT47b9jh071K9fP+Xl5alWrVpyOBzaunWrzpw5o1tvvdXj49IBAQAAAMohu92ufv36acqUKQoODladOnU0efJkhYaGqkePHiosLNTp06eLOx2NGzfWwYMH9corr+jRRx/ViRMnNHbsWD300EOqW7eux8elAAEAAADKqZEjR6qgoEDjxo1TTk6O2rRpo9mzZ8tutyshIUHdunXTq6++qvvvv1/Vq1fXe++9p1dffVX33HOPatSooYceekhDhw69omPanE6n06Cfp9RpPXSq1SEAAADgEnbOGmV1CBf14UHPx4zM1D9ik9UhXBHOAQEAAABgGgoQAAAAAKbhHBAAAADAA4VOvrv3BrIIAAAAwDQUIAAAAABMwwgWAAAA4IEiXf1N//BfdEAAAAAAmIYCBAAAAIBpGMECAAAAPMBVsLyDLAIAAAAwDQUIAAAAANMwggUAAAB4oJDv7r2CLAIAAAAwDQUIAAAAANMwggUAAAB4oMjJjQi9gQ4IAAAAANNQgAAAAAAwDSNYAAAAgAe4CpZ3kEUAAAAApqEAAQAAAGAaRrAAAAAADxQ5+e7eG8giAAAAANNQgAAAAAAwDSNYAAAAgAcKxY0IvYEOCAAAAADTUIAAAAAAMA0jWAAAAIAHuAqWd5BFAAAAAKahAAEAAABgGkawAAAAAA9wFSzvoAMCAAAAwDQUIAAAAABMwwgWAAAA4AGuguUdZBEAAACAaShAAAAAAJiGESwAAADAA4WMYHkFWQQAAABgGgoQAAAAAKZhBAsAAADwQBE3IvQKOiAAAAAATEMBAgAAAMA0jGABAAAAHuAqWN5BFgEAAACYhgIEAAAAgGkYwQIAAAA8UOTkKljeQAcEAAAAgGkoQAAAAACYhhEsAAAAwAOFfHfvFRQgFrk1qr6G3dtBDcOClXY2W4vX79bcNdsuun2D2jW0dMIAt/UjSafVe+KHBkbqu8ixsciv8cix8cixsciv8cgxfBEFiAVaNArT1KG99NWOA5r52Ua1alJHw+/toAo2m2av3lriPk3r1pIkxb75qfIKCorXc/IKSty+vCPHxiK/xiPHxiPHxiK/xiPH8FUUIBYYfHd77U9I1fPzVkuSNsUdU6UKFTTgzjaav26HcvML3fZpFh6ik7+ka8fBBLPD9Unk2Fjk13jk2Hjk2Fjk13jk2HxcBcs7GGQzWeVKFXVzRLi+2XXIZf3rXQfl77ArukmdEvdrFl5L+xNSzQjR55FjY5Ff45Fj45FjY5Ff45Fj+DI6ICYLr1lN9sqVdDzljMv6idQ0SVL9kBravO+4235Nw2vpSNIpzRvzoJrVDdHZrFx9vjlOsz7bpIKiIjNC9xnk2Fjk13jk2Hjk2Fjk13jkGL6MAsRkgVX8JEmZOXku61m/PvevYnfbJziwqmpW85fT6dS0ZRuUeCZDbZvV04A7YlS7RoDGzV1tfOA+hBwbi/wajxwbjxwbi/wajxxbo4jhIa+gADGZzfbr7KDTWeLrJX35cC4nV0PeXqJjyaeVfCZTkrTz4EnlFxRqeK8Omr1qq44knTYqZJ9Djo1Ffo1Hjo1Hjo1Ffo1HjuHLKONMdjY7V5Lk73D9ZqLqr88zc3Ld9snNL9TW+OPFvywuWL/niCQpIrymEaH6LHJsLPJrPHJsPHJsLPJrPHIMX2Z5B+SRRx75bxV/GR999JHB0RgvITVNBYVFqhtS3WW9bq3zz39KPOW2T/3aNRTTtK5Wb4vXuf9ptToqn/+fLy0zx7B4fRE5Nhb5NR45Nh45Nhb5NR45tkYhV8HyCss7ILfccou2bdumU6dOqU6dOpd8lAV5BYXadShBt7dq4rLePTpCGVk52ns0yW2fkOoBeu6P3dQ9OsJl/Y6YpsrMztW+48mGxuxryLGxyK/xyLHxyLGxyK/xyDF8meUdkGHDhqlq1aqaNm2a/v73vys8PNzqkAz3waqtmjWyt14fdLdWfLdXLRuF6U89YjRt+Xrl5hfK32FXo7BgnUhNV1pmtnYcSNC2/Sc0uk9nOfwq62jSaXVq3lAP3Ratt5Z+q7NZ7m3W8o4cG4v8Go8cG48cG4v8Go8cw1fZnM6LnL1kskGDBql69eqaMmWKYcdoPXSqYe99pbq2bKwhPW9R/do1lJJ+Tp/8+3vNX7dTknRzRLjeH/2Axn+4Rp9vjpMkBTjsGtzzFt3WsrFqVvNXQmq6FnyzU8s27rHyxyjVyLGxyK/xyLHxyLGxyK/xymKOd84aZXUIF/WXXX2tDqFEb0f/0+oQrkipKUCSk5MVFxenrl27GnaM0lSAAAAAwB0FyJXztQLE8hGsC2rXrq3atWtbHQYAAAAAA5WaAgQAAAAozYqcll+/qUwgiwAAAABMQwECAAAAwDSMYAEAAAAeKBQ3IvQGOiAAAAAATEMBAgAAAMA0jGABAAAAHihyMoLlDXRAAAAAAJiGAgQAAACAaRjBAgAAADzAjQi9gywCAAAAMA0FCAAAAADTMIIFAAAAeKCIGxF6BR0QAAAAAKahAAEAAABgGkawAAAAAA8UciNCr6ADAgAAAMA0FCAAAAAATMMIFgAAAOABbkToHWQRAAAAgGkoQAAAAACYhhEsAAAAwANFXAXLK+iAAAAAADANBQgAAAAA0zCCBQAAAHigSIxgeQMdEAAAAACmoQABAAAAYBpGsAAAAAAPcBUs76ADAgAAAJRTRUVFmjZtmjp16qSWLVtq4MCBOnbs2EW3T01N1ejRo9WuXTu1a9dOf/nLX5SUlHRFx6QAAQAAAMqpmTNnauHChZo0aZIWLVokm82m2NhY5eXllbj9qFGjlJiYqLlz52ru3LlKSkrSsGHDruiYFCAAAACAB4qcFUrl42rl5eVpzpw5GjFihLp06aLIyEhNnTpVycnJWrt2rdv2GRkZ2rZtm2JjYxUVFaWoqCg99thj2rt3r86cOePxcSlAAAAAgHIoPj5e586dU/v27YvXgoKCFBUVpW3btrlt7+fnp6pVq2r58uXKzMxUZmamVqxYoQYNGqhatWoeH5eT0AEAAIBy6MK5G2FhYS7rISEhSkxMdNvez89PL7/8sl588UXFxMTIZrOpVq1amj9/vipU8LyvQQECAAAAeKC0XgWrW7dul3x93bp1Ja5nZ2dLkux2u8u6n5+f0tPT3bZ3Op3av3+/oqOjNWjQIBUWFmrq1KkaPny4/vnPfyogIMCjeClAAAAAgHLI4XBIOn8uyIU/S1Jubq6qVKnitv2XX36pBQsW6F//+ldxsfHuu++qa9euWrJkifr37+/RcSlAAAAAAB92sQ7H5VwYvUpJSVG9evWK11NSUhQZGem2/Y4dO9SwYUOXTke1atXUsGFDHT161OPjchI6AAAA4IEi2Url42pFRkYqICBAW7ZsKV7LyMhQXFycYmJi3LYPCwvTsWPHlJubW7yWnZ2thIQE1a9f3+PjUoAAAAAA5ZDdble/fv00ZcoUrVu3TvHx8Ro1apRCQ0PVo0cPFRYWKjU1VTk5OZKk++67T5L0xBNPKD4+vnh7u92u+++/3+PjUoAAAAAA5dTIkSPVp08fjRs3Tn379lXFihU1e/Zs2e12JSYmqmPHjlq5cqWk81fHWrBggZxOp/r3768///nPqly5sv75z38qKCjI42PanE6n06gfqLRpPXSq1SEAAADgEnbOGmV1CBf1h43DrQ6hRMs6vGN1CFeEDggAAAAA01CAAAAAADANl+EFAAAAPFBab0Toa+iAAAAAADANBQgAAAAA0zCCBQAAAHiAESzvoAMCAAAAwDQUIAAAAABMwwgWAAAA4AFGsLyDDggAAAAA01CAAAAAADANI1gAAACAB4rECJY30AEBAAAAYBoKEAAAAACmYQQLAAAA8ABXwfIOOiAAAAAATEMBAgAAAMA0jGABAAAAHmAEyzvogAAAAAAwDQUIAAAAANMwggUAAAB4gBEs76ADAgAAAMA0FCAAAAAATMMIFgAAAOABRrC8gw4IAAAAANNQgAAAAAAwDSNYAAAAgAecjGB5BR0QAAAAAKahAAEAAABgGkawAAAAAA8UiREsb6ADAgAAAMA0FCAAAAAATMMIFgAAAOABbkToHXRAAAAAAJiGAgQAAACAaRjBAgAAADzAjQi9gw4IAAAAANNQgAAAAAAwDSNYAAAAgAe4CpZ30AEBAAAAYBoKEAAAAACmYQQLAAAA8ABXwfIOOiAAAAAATEMBAgAAAMA0jGABAAAAHuAqWN5BBwQAAACAaShAAAAAAJiGESwAAADAA06n1RGUDXRAAAAAAJiGAgQAAACAaRjBAgAAADxQJK6C5Q10QAAAAACYhgIEAAAAgGkYwQIAAAA84ORGhF5BBwQAAACAaShAAAAAAJiGESyL3BpVX8Pu7aCGYcFKO5utxet3a+6abRfdvkHtGlo6YYDb+pGk0+o98UMDI/Vd5NhY5Nd45Nh45NhY5Nd45NhcRYxgeQUFiAVaNArT1KG99NWOA5r52Ua1alJHw+/toAo2m2av3lriPk3r1pIkxb75qfIKCorXc/IKSty+vCPHxiK/xiPHxiPHxiK/xiPH8FUUIBYYfHd77U9I1fPzVkuSNsUdU6UKFTTgzjaav26HcvML3fZpFh6ik7+ka8fBBLPD9Unk2Fjk13jk2Hjk2Fjk13jkGL6Kc0BMVrlSRd0cEa5vdh1yWf9610H5O+yKblKnxP2ahdfS/oRUM0L0eeTYWOTXeOTYeOTYWOTXeOTYGk5n6Xz4GjogJguvWU32ypV0POWMy/qJ1DRJUv2QGtq877jbfk3Da+lI0inNG/OgmtUN0dmsXH2+OU6zPtukgqIiM0L3GeTYWOTXeOTYeOTYWOTXeOQYvszyAuTIkSP64osvlJ6erk6dOqlLly4ur2dmZurll1/Wq6++alGE3hVYxU+SlJmT57Ke9etz/yp2t32CA6uqZjV/OZ1OTVu2QYlnMtS2WT0NuCNGtWsEaNzc1cYH7kPIsbHIr/HIsfHIsbHIr/HIMXyZpQXIjh079Oijj6p27dpyOp36+OOP1b17d73xxhuy28//xcnJydHy5cvLTAFis/169YSL9MtK+vLhXE6uhry9RMeSTyv5TKYkaefBk8ovKNTwXh00e9VWHUk6bVTIPoccG4v8Go8cG48cG4v8Go8cW4MbEXqHpeeAvPHGG+rTp4/WrFmjr776Sm+++aY2btyoIUOGKD8/38rQDHM2O1eS5O9w/Wai6q/PM3Ny3fbJzS/U1vjjxb8sLli/54gkKSK8phGh+ixybCzyazxybDxybCzyazxyDF9maQGyf/9+9evXr/j5XXfdpffff1+7du3S2LFjLYzMOAmpaSooLFLdkOou63VrnX/+U+Ipt33q166h3p1auP2ScVQ+38BKy8wxJFZfRY6NRX6NR46NR46NRX6NR47hyywtQAICAnTmjOvJUzfffLMmT56sNWvWlJmxq/+VV1CoXYcSdHurJi7r3aMjlJGVo71Hk9z2CakeoOf+2E3doyNc1u+IaarM7FztO55saMy+hhwbi/wajxwbjxwbi/wajxxbw+m0lcqHr7H0HJAuXbroxRdf1IQJExQVFaXKlStLkrp3765nn31WkyZNUmJiopUhGuKDVVs1a2RvvT7obq34bq9aNgrTn3rEaNry9crNL5S/w65GYcE6kZqutMxs7TiQoG37T2h0n85y+FXW0aTT6tS8oR66LVpvLf1WZ7Pc26zlHTk2Fvk1Hjk2Hjk2Fvk1HjmGr7I5ndZdPTg9PV2jRo3Sd999p7///e/q3Lmzy+sLFizQK6+8osLCQu3bt++aj9d66NRrfg9v6dqysYb0vEX1a9dQSvo5ffLv7zV/3U5J0s0R4Xp/9AMa/+Eafb45TpIU4LBrcM9bdFvLxqpZzV8Jqela8M1OLdu4x8ofo1Qjx8Yiv8Yjx8Yjx8Yiv8YrizneOWuU1SFcVPPPxlsdQol+vHei1SFcEUsLkAuOHz+uGjVqKDAw0O21I0eO6KuvvtLgwYOv+TilqQABAACAu9JcgNy4YoLVIZRob68JVodwRSy/D4gk1atX76KvNWzY0CvFBwAAAADrWXoSOgAAAIDypVR0QAAAAIDSzvoTF8oGOiAAAAAATEMBAgAAAMA0jGABAAAAHvDFm/6VRnRAAAAAAJiGAgQAAACAaRjBAgAAADzACJZ30AEBAAAAYBoKEAAAAACmYQQLAAAA8AD3IfQOOiAAAAAATEMBAgAAAMA0jGABAAAAHuAqWN5BBwQAAACAaShAAAAAAJiGESwAAADAE1wGyyvogAAAAAAwDQUIAAAAANMwggUAAAB4gKtgeQcdEAAAAACmoQABAAAAyqmioiJNmzZNnTp1UsuWLTVw4EAdO3asxG2nT5+uZs2alfh45plnPD4mBQgAAADgAaezdD6uxcyZM7Vw4UJNmjRJixYtks1mU2xsrPLy8ty2HThwoDZs2ODyeOKJJ+RwONS/f3+Pj0kBAgAAAJRDeXl5mjNnjkaMGKEuXbooMjJSU6dOVXJystauXeu2vb+/v2rVqlX8yM7O1t///nc9/fTTioyM9Pi4FCAAAABAORQfH69z586pffv2xWtBQUGKiorStm3bLrv/a6+9poiICD344INXdFyuggUAAAB4oLReBatbt26XfH3dunUlriclJUmSwsLCXNZDQkKUmJh4yff88ccftW7dOn344YeqUOHKehp0QAAAAIByKDs7W5Jkt9td1v38/JSbm3vJfefNm6eWLVu6dE88RQcEAAAA8GEX63BcjsPhkHT+XJALf5ak3NxcValS5aL7ZWVlae3atRo/fvxVHZcOCAAAAOAJp610Pq7ShdGrlJQUl/WUlBSFhoZedL/169erqKhIPXr0uKrjUoAAAAAA5VBkZKQCAgK0ZcuW4rWMjAzFxcUpJibmovvt2LFDN954o4KCgq7quIxgAQAAAOWQ3W5Xv379NGXKFAUHB6tOnTqaPHmyQkND1aNHDxUWFur06dMKDAx0GdGKj49X06ZNr/q4dEAAAAAAD1h9w0EjbkQ4cuRI9enTR+PGjVPfvn1VsWJFzZ49W3a7XYmJierYsaNWrlzpss8vv/yi6tWrX/UxbU7ntYbtO1oPnWp1CAAAALiEnbNGWR3CRTX65ytWh1Cin/o+a3UIV4QOCAAAAADTcA4IAAAA4IlyMzdkLDogAAAAAExDAQIAAADANIxgAQAAAB5wXsNN//BfdEAAAAAAmIYCBAAAAIBpGMECAAAAPMFVsLyCDggAAAAA01CAAAAAADANI1gAAACAB7gKlnfQAQEAAABgGgoQAAAAAKZhBAsAAADwBFfB8go6IAAAAABMU646INf9fZPVIZRp6SsjrA6hzCtYUcvqEMq0Sr1SrQ6hzOMzDAAoVwUIAAAAcPW4CpY3MIIFAAAAwDQUIAAAAABMwwgWAAAA4AmuguUVdEAAAAAAmIYCBAAAAIBpGMECAAAAPMEIllfQAQEAAABgGgoQAAAAAKZhBAsAAADwhJMbEXoDHRAAAAAApqEAAQAAAGAaRrAAAAAADzi5CpZX0AEBAAAAYBoKEAAAAACmYQQLAAAA8AQjWF5BBwQAAACAaShAAAAAAJiGESwAAADAE9yI0CvogAAAAAAwDQUIAAAAANMwggUAAAB4wMZVsLyCDggAAAAA01CAAAAAADANI1gAAACAJxjB8go6IAAAAABMQwECAAAAwDSMYAEAAACe4EaEXkEHBAAAAIBpKEAAAAAAmIYRLAAAAMATXAXLK+iAAAAAADANBQgAAAAA0zCCBQAAAHiCESyvoAMCAAAAwDQUIAAAAABMwwgWAAAA4AlGsLyCDggAAAAA01CAAAAAADANI1gAAACAJ5w2qyMoE+iAAAAAADANBQgAAAAA0zCCBQAAAHjAxlWwvIIOCAAAAADTUIAAAAAAMA0jWAAAAIAnGMHyCjogAAAAAExDB8QkMXe20p9fekj1osKVnpqhL/7+lRa+tvyi21eqXEl9nuypHn+6TbXqXqdfEk7rmwXrtfC15SrILyjebtHP7ys4tLrb/v8XFqszyWne/0FKqfY1IzQ0oocaBoToTN45LT2xVR/+9J8St727TmuNb97nou81cfen+vLnXZKkVV2f0XV+gW7b3PXNKzqVl+md4H3UrVH1NezeDmoYFqy0s9lavH635q7ZdtHtG9SuoaUTBritH0k6rd4TPzQwUt/AZ9gafI6NRX6NR47hiyhATBB1S1O9uOIp/WfRJs19fqFu6hipP0/qqwoVKmjBK0tL3GfoWwPU409d9PGkJTqw7ZCatG6kR8Y/oJD6tfTmoFmSpOoh1RQcWl2zRs/Tvu8OuOyfceqs4T9XadG8ej290foRrU38UbMOrlWrGg00NKKHKsimuT/92237jSn7NfC7WS5rNpv07I33y7+Snzamns9lsD1A1/kFauq+L/Vj2nGX7dPyswz7eXxBi0Zhmjq0l77acUAzP9uoVk3qaPi9HVTBZtPs1VtL3Kdp3VqSpNg3P1VewX+L6Jy8ghK3L0/4DFuDz7GxyK/xyDF8FQWICR554QEd/v6oXu8/XZK0fc33qlS5oh586j4tfvML5eXkuWwfWCNAPQf30AdPf6xPp3wmSdr1zR5J0mN/e0Szn/5Y6b9kqEl0Q0nShqVblHL8FxN/otIltsntOpCRqAk/fipJ2vzLQVWyVdCfGnXRgqMblFvk+ks1Lf+c0tLPuaw9WP9WNQiopUGb31Va/vnXmgaFSZL+lbxXSTlpxv8gPmTw3e21PyFVz89bLUnaFHdMlSpU0IA722j+uh3KzS9026dZeIhO/pKuHQcTzA631OMzbA0+x8Yiv8Yjx/BVV3QOSGpqqiZMmKBHH31UTz31lObNm6ft27crOzvbqPh8XmV7JbW47UZtWLbFZf3bxZtVNbCKmneKdNvHv1pVffH3tfrus+0u6wkHfpYkhTUKkSQ1btVAZ89kluvio7KtoloHN9K/kve6rK9L2iP/Sn5qVaPBZd/jOnuAhkT00JLjW7Q3/b+/kJsGhikjP5t/uP1G5UoVdXNEuL7Zdchl/etdB+XvsCu6SZ0S92sWXkv7E1LNCNGn8Bm2Bp9jY5Ff45Fj+LIr6oA8++yz2rBhgyIiIpSQkKDPP/9cTqdTFSpUUKNGjXTTTTepefPmat68uSIjI1W5cmWj4vYZYY1qy+5XWSd/LR4u+PlQkiQpvOn12rF2t8trSUdTNH34B27v1fEP7ZSfV6CEA4mSpMYtGyjzzDmNX/xXRXdrrgoVK2jzFzv07uh5Op2UZswPVMrUqRose4VKOp51ymU94dfn9fxrasupQyXtWuyxiO4qchbp3YNrXdabBl2vs/nZer3Vw2pzXWNVsNm0IXW/psZ/qVO55WfE7bfCa1aTvXIlHU8547J+IjVNklQ/pIY27zvutl/T8Fo6knRK88Y8qGZ1Q3Q2K1efb47TrM82qaCoyIzQSyU+w9bgc2ws8ms8cmwNbkToHVdUgOzatUtjxozRwIEDJUlZWVnau3evfvzxR/3444/atm2bli1bJkmy2+3avXv3pd5OkpSbm6uDBw+qSZMmcjgc2rdvn+bPn6/k5GRFRESof//+Cg0NvYofrXTwr+4vSTqX4dolyjp7/nnVoCoevU/H+9up+yOdtWzaSmWmnR+vaNyqgWqGX6eVH3ytJW99oXo3hKv/xAf1xr8namjrscrJyvXiT1I6BVZ2SJLOFeS4rGcVnh9r86/kd8n9a9j99fvro/Xx0Q3K/M17NA0MU4gjSMsTtumfxzaqgX8tDY7orr+3jVW/TdOVU5jvxZ/EdwRWOZ/TzN+MDmb9+ty/it1tn+DAqqpZzV9Op1PTlm1Q4pkMtW1WTwPuiFHtGgEaN3e18YGXUnyGrcHn2Fjk13jkGL7sigoQPz8/RUVFFT+vWrWq2rRpozZt2hSvpaWlaffu3dqzZ89l3+/w4cMaMGCAUlNTdf3112vSpEkaNmyYwsPD1bhxY3399ddaunSpFixYoMaNG19JqKVGhQq2839wllwyFxVdvpTu1Lu9np4/Urv/E6fZT39cvD5l4Ezl5eTp8PdHJUl7NsTr2N4TemvDJHX/Uxd98e5X1xx/aWfThfyW/HrRRfJ+wX3hbVTBZtPCo5vcXntpz2LlFhbowNnzHafvzxzVT5nJ+qD9EN19fWstObHFbZ/ywGa73Gfafe1cTq6GvL1Ex5JPK/nM+Ssv7Tx4UvkFhRreq4Nmr9qqI0mnjQq5VOMzbA0+x8Yiv8Yjx/BlV3QOSPfu3RUXF3fJbapXr67OnTtr2LBhl32/v/3tb4qOjtby5ct18803a+jQobrnnnv0+eef6+2339aqVavUoUMHvfrqq1cSZqlyoVtRNaiqy3rVwPOdj3Ppl74STe9RPfXcwlHauyFez9/7uvL/5yoV+zYfKC4+Lti7ab8y086pccv6Xoi+9Lvwje9vvyWuWvH8Nz/nCi7dBbo99CZt+eVQ8Um7/+vHtBPF/3C7YHfacZ3Nz1ZEoO925a7V2ezzOfV3uH67VvXX55k57jnPzS/U1vjjxf/Bu2D9niOSpIjwmkaE6hP4DFuDz7GxyK/xyLFFnLbS+fAxV1SA9O7dW6tWrdKhQ5eeR/bU1q1b9cQTTygyMlJPPfWUcnNz1bdv3+KqvlKlShoyZIh27NjhleNZ4efDySosKNT1TVz/Y3/h+fG4i1+FYvi0gRryRn+tX7xZz939inLO/Xe8wr9aVd35566qd0O4236V7JWU/kv5mO9OyDqtgqJChVe9zmX9wvOfMlMuum+IX5CaBV2vr5N+dHstoJJD99S5WQ39Q9xeq1yhYrm+hGlCapoKCotUN6S6y3rdWuef/5R4ym2f+rVrqHenFm7/oXRUPt+ETcvMcdunvOAzbA0+x8Yiv8Yjx/BlV1SA/N///Z/27NmjBx54QM8884xWrlypY8eOXfXBHQ6HcnLOf9hr1qyp//u//5Ofn+u3gBkZGQoMdL+Jlq/Iz83X7m/3qeMf2rmsd+7TXmfPZCp+a8nF3MBX/qj7Hr9Li6d+oZf7TnXpfEhSQV6BRr4zSA89dZ/L+q292shR1U8//Nv1ijplVV5Rgb4/c1RdQ290We8WepMy8rMVl37iovtGVa8rSfrhjPtnOL+oUGOj7lX/Rp1d1ruE3CBHRbt2nP7JC9H7pryCQu06lKDbWzVxWe8eHaGMrBztPZrktk9I9QA998du6h4d4bJ+R0xTZWbnat/xZENjLs34DFuDz7GxyK/xyDF82RWdAzJp0iTt27dPe/fu1apVq7Rs2TLZbDb5+/srKipKN910k8aOHevx+3Xs2FEvvfSSJk2apMaNG+vFF18sfs3pdGrr1q2aOHGiunfvfiVhljoLXl6i19c+r+cXjdbqud8o6tZmeuCv9+qDpz9WXk6eqgZWUf2ocP18OFnpv2SoccsGenBsL+3fdkjffrJJN7Rz/UVxLC5BWWez9cnkz9Tv+T46k5Ku7at3qWGL+vrT+P/T5i92aNc6929Ey6o5h/+lGW0G6tVWffVZwg61qF5P/Rp20oz9a5RbVCD/in5qGBCihKzTLmMqTQJqK7cwXyez3eddc4vy9Y8j6zWoye06lZepzakH1SQwVLFNumlDSry2nTps5o9Y6nywaqtmjeyt1wfdrRXf7VXLRmH6U48YTVu+Xrn5hfJ32NUoLFgnUtOVlpmtHQcStG3/CY3u01kOv8o6mnRanZo31EO3Reutpd/qbDm4YMKl8Bm2Bp9jY5Ff45FjC3AVLK+wOZ2XOcPxIoqKinT48GHt3btXe/bsUVxcnOLj47Vz506P3+P06dMaMmSI6tatqzfeeMPltS+//FJPPvmkOnXqpKlTpyogIOBqwnTRo8ID1/weV6vDfW31pwn/p/Bm1+vUydP6bOZqLX7zC0lSiy5ReuNfEzX5z+/oqw//rf4TH1S/5/tc9L2e7Dpeu/8TJ5vNpnuG3qGeQ+7Q9Y1DlXHqrL755wZ9NP4Tt5sbmiF9ZcTlNzLIbSFRio3opvr+tZSak6FPj2/WgqMbJEmtgxvq3baxmvjjYn158r+fz7FR96pr7Rt1179KPsfIJpt612un3nXbqU7VYKXnZ+mrn3/Qe4e+drsxnFkKVtSy5Lgl6dqysYb0vEX1a9dQSvo5ffLv7zV/3fn83hwRrvdHP6DxH67R55vPnzcW4LBrcM9bdFvLxqpZzV8Jqela8M1OLdt4+QtWmKVSL+uujc9n2Bpl8XNcmpBf45XFHO+cNcrqEC6q0VtvWh1CiX56YrTVIVyRqy5ASuJ0Ov97VYYrkJaWpurVq7usnT59WikpKYqMdL9R39WysgApD6wsQMqL0vaPt7LGygKkvOAzDOByKECunK8VIFc0gnU5V1N8SHIrPiQpODhYwcHB1xgRAAAA4CWMYHnFFZ2EDgAAAADXggIEAAAAgGm8OoIFAAAAlFU2RrC8gg4IAAAAANNQgAAAAAAwDSNYAAAAgCcYwfIKOiAAAAAATEMBAgAAAMA0jGABAAAAnmAEyyvogAAAAAAwDQUIAAAAANMwggUAAAB4gBsRegcdEAAAAACmoQABAAAAYBoKEAAAAMATTlvpfFyDoqIiTZs2TZ06dVLLli01cOBAHTt27KLb5+fn64033lCnTp3UqlUr9evXT/v27buiY1KAAAAAAOXUzJkztXDhQk2aNEmLFi2SzWZTbGys8vLyStx+woQJWrx4sV566SUtWbJE1atXV2xsrM6ePevxMSlAAAAAgHIoLy9Pc+bM0YgRI9SlSxdFRkZq6tSpSk5O1tq1a922P3HihBYvXqxXX31Vt912mxo3bqxXXnlFdrtde/bs8fi4FCAAAACAJ5yl9HGV4uPjde7cObVv3754LSgoSFFRUdq2bZvb9hs2bFBQUJA6d+7ssv0333yjW265xePjchleAAAAwId169btkq+vW7euxPWkpCRJUlhYmMt6SEiIEhMT3bY/evSo6tatq6+++krvvfeekpOTFRUVpaefflqNGzf2OF46IAAAAEA5lJ2dLUmy2+0u635+fsrNzXXbPjMzU8ePH9fMmTM1evRozZo1S5UqVdIf//hHnTp1yuPj0gEBAAAAPFBab0R4sQ7H5TgcDknnzwW58GdJys3NVZUqVdy2r1y5ss6ePaupU6cWdzymTp2qLl26aNmyZRo0aJBHx6UDAgAAAJRDF0avUlJSXNZTUlIUGhrqtn1oaKgqVarkMm7lcDhUt25dJSQkeHxcChAAAACgHIqMjFRAQIC2bNlSvJaRkaG4uDjFxMS4bR8TE6OCggL9+OOPxWs5OTk6ceKE6tev7/FxGcECAAAAPFFKR7Cult1uV79+/TRlyhQFBwerTp06mjx5skJDQ9WjRw8VFhbq9OnTCgwMlMPhUExMjG699VY99dRTevHFF1W9enVNmzZNFStWVK9evTw+Lh0QAAAAoJwaOXKk+vTpo3Hjxqlv376qWLGiZs+eLbvdrsTERHXs2FErV64s3n769Olq27atHn/8cfXp00eZmZn66KOPFBwc7PExbU6ns4zVchfXo8IDVodQpqWvjLA6hDKvYEUtq0Mo0yr1SrU6hDKPzzCAy9k5a5TVIVxUxKtTrQ6hRAefKb05KwkjWAAAAIAHSutVsHwNI1gAAAAATEMBAgAAAMA0jGABAAAAnmAEyyvogAAAAAAwDQUIAAAAANMwggUAAAB4ghEsr6ADAgAAAMA0FCAAAAAATMMIFgAAAOABbkToHXRAAAAAAJiGAgQAAACAaShAAAAAAJiGAgQAAACAaShAAAAAAJiGq2ABAAAAnuAqWF5BBwQAAACAaShAAAAAAJiGESwAAADAA9yI0DvogAAAAAAwDQUIAAAAANMwggUAAAB4ghEsryhXBcipwbdaHULZtsLqAMq+7eNnWR0CcE1iVgy1OgQAgMUYwQIAAABgmnLVAQEAAACuGiNYXkEHBAAAAIBpKEAAAAAAmIYRLAAAAMAD3IjQO+iAAAAAADANBQgAAAAA0zCCBQAAAHiCESyvoAMCAAAAwDQUIAAAAABMwwgWAAAA4AGuguUddEAAAAAAmIYCBAAAAIBpGMECAAAAPMEIllfQAQEAAABgGgoQAAAAAKZhBAsAAADwBCNYXkEHBAAAAIBpKEAAAAAAmIYRLAAAAMAD3IjQO+iAAAAAADANBQgAAAAA0zCCBQAAAHiCESyvoAMCAAAAwDQUIAAAAABMwwgWAAAA4AlGsLyCDggAAAAA01CAAAAAADANI1gAAACAB7gRoXfQAQEAAABgGgoQAAAAAKZhBAsAAADwBCNYXkEHBAAAAIBpKEAAAAAAmIYRLAAAAMADXAXLO+iAAAAAADANBQgAAAAA0zCCBQAAAHiCESyvoAMCAAAAwDQUIAAAAABMwwgWAAAA4AlGsLyCDggAAAAA01CAAAAAADANBQgAAAAA03AOCAAAAOABm9UBlBF0QAAAAACYhgIEAAAAgGkYwbLIrVH1NezeDmoYFqy0s9lavH635q7ZdtHtG9SuoaUTBritH0k6rd4TPzQwUt9Fjq2XmCL1+rM0Y5LUNtrqaMomcnxt+D1hLPJrPHJsMi7D6xUUIBZo0ShMU4f20lc7DmjmZxvVqkkdDb+3gyrYbJq9emuJ+zStW0uSFPvmp8orKChez8krKHH78o4cW+9kkhQ7RjqbaRO/sY1Bjq8NvyeMRX6NR47hqyhALDD47vban5Cq5+etliRtijumShUqaMCdbTR/3Q7l5he67dMsPEQnf0nXjoMJZofrk8ixdYqKpOWrpb/NsjqSsoscewe/J4xFfo1HjuGrOAfEZJUrVdTNEeH6Ztchl/Wvdx2Uv8Ou6CZ1StyvWXgt7U9INSNEn0eOrbX/sDRxqnTfndLrz1kdTdlEjq8dvyeMRX6NR46tYXOWzoevoQNisvCa1WSvXEnHU864rJ9ITZMk1Q+poc37jrvt1zS8lo4kndK8MQ+qWd0Qnc3K1eeb4zTrs00qKCoyI3SfQY6tFVZbWvOxFBoibd1ldTRlEzm+dvyeMBb5NR45hi8rtQXIPffco/fee09hYWFWh+JVgVX8JEmZOXku61m/PvevYnfbJziwqmpW85fT6dS0ZRuUeCZDbZvV04A7YlS7RoDGzV1tfOA+hBxbq3qQpCCroyjbyPG14/eEsciv8cgxfJmlBcjy5csv+tqxY8e0atUqBQcHS5Luu+8+c4IymM326y1snCX3y0r68uFcTq6GvL1Ex5JPK/lMpiRp58GTyi8o1PBeHTR71VYdSTptVMg+hxwDuBx+TxiL/BqPHFvEB8edSiNLC5CJEycqJydHkuQs4S/Q3/72N0nn/5KVlQLkbHauJMnf4frNRNVfn2fm5Lrtk5tfqK3x7m3U9XuOaHivDooIr8kvjP9BjgFcDr8njEV+jUeO4cssPQl96dKlioqKUrt27fSf//xH8fHxxY8qVapo7dq1io+P1759+6wM06sSUtNUUFikuiHVXdbr1jr//KfEU2771K9dQ707tXD7JeOofL5+TMvMMSRWX0WOAVwOvyeMRX6NR47hyywtQBo2bKhFixapRYsW6tWrl1auXGllOKbIKyjUrkMJur1VE5f17tERysjK0d6jSW77hFQP0HN/7Kbu0REu63fENFVmdq72HU82NGZfQ44BXA6/J4xFfo1Hji3iLKUPH2P5SeiVKlXS6NGj1alTJz311FNat26dJkyYYHVYhvpg1VbNGtlbrw+6Wyu+26uWjcL0px4xmrZ8vXLzC+XvsKtRWLBOpKYrLTNbOw4kaNv+Exrdp7McfpV1NOm0OjVvqIdui9ZbS7/V2Sz3Nmt5R44BXA6/J4xFfo1HjuGrbM6STr6wSEZGhiZOnKjt27fr1KlTWrVqlerWreu19289dKrX3utadW3ZWEN63qL6tWsoJf2cPvn395q/bqck6eaIcL0/+gGN/3CNPt8cJ0kKcNg1uOctuq1lY9Ws5q+E1HQt+Ganlm3cY+WPUaqVxRxvH+9bd57bukvq/4RNH77lVNtoq6Mpm3wtxzETh1odgouy+HuiNCG/xiuLOd45a5TVIVxUy5Gl59+S/+uHaaU3ZyUpVQXIBcuXL9fSpUs1ZcoUhYSEeO19S1MBAlwNXytAgN8qbQUIgNKnNBcgrUaUzn9Lfj+99OasJJaPYJXkvvvuKzNXvQIAAADwX5aehA4AAACgfCmVHRAAAACg1Cl1Jy74JjogAAAAAExDAQIAAADANIxgAQAAAB6wMYLlFXRAAAAAAJiGAgQAAAAop4qKijRt2jR16tRJLVu21MCBA3Xs2LGLbr9s2TI1a9bM7XGpfX6LESwAAADAE2VwBGvmzJlauHChXn31VdWuXVuTJ09WbGysvvjiC9ntdrft9+/fr7Zt2+rNN990WQ8ODvb4mHRAAAAAgHIoLy9Pc+bM0YgRI9SlSxdFRkZq6tSpSk5O1tq1a0vc58CBA4qMjFStWrVcHhUrVvT4uBQgAAAAQDkUHx+vc+fOqX379sVrQUFBioqK0rZt20rcZ//+/WrSpMk1HZcCBAAAAPCAzVk6H1crKSlJkhQWFuayHhISosTERLftT58+rV9++UXbtm1Tz5491bFjRw0fPlxHjhy5ouNyDggAAADgw7p163bJ19etW1fienZ2tiS5nevh5+en9PR0t+0PHDggSapYsaJef/11ZWVlaebMmfrjH/+ozz//XDVr1vQoXgoQAAAAoBxyOBySzp8LcuHPkpSbm6sqVaq4bd++fXtt3bpV1apVK15755131LVrVy1dulSPPfaYR8elAAEAAAA8UUqvgnWxDsflXBi9SklJUb169YrXU1JSFBkZWeI+/1t8SFLVqlUVHh6u5ORkj4/LOSAAAABAORQZGamAgABt2bKleC0jI0NxcXGKiYlx237BggVq166dcnJyitcyMzN19OjRKzoxnQIEAAAAKIfsdrv69eunKVOmaN26dYqPj9eoUaMUGhqqHj16qLCwUKmpqcUFR9euXeV0OjV27FgdPHhQP/74o0aMGKHg4GD94Q9/8Pi4FCAAAACAJ5yl9HENRo4cqT59+mjcuHHq27evKlasqNmzZ8tutysxMVEdO3bUypUrJZ0f2frwww917tw59e3bVwMGDFBgYKA++ugjl3NILodzQAAAAIByqmLFihozZozGjBnj9lp4eLj279/vsnbDDTdo9uzZ13RMOiAAAAAATEMHBAAAAPDAtdz0D/9FBwQAAACAaShAAAAAAJiGESwAAADAE4xgeQUdEAAAAACmoQABAAAAYBpGsAAAAAAP2JzMYHkDHRAAAAAApqEAAQAAAGAaRrAAAAAATzCB5RV0QAAAAACYhgIEAAAAgGkYwQIAAAA8YGMEyyvogAAAAAAwDQUIAAAAANMwggUAAAB4ghEsr6ADAgAAAMA0FCAAAAAATMMIFgAAAOABroLlHXRAAAAAAJiGAgQAAACAaRjBAgAAADzBCJZX0AEBAAAAYBoKEAAAAACmYQQLAAAA8ABXwfIOOiAAAAAATEMBAgAAAMA0jGABAAAAnmAEyyvogAAAAAAwTbnqgFTqlWp1CMA1af99H6tDKNMKVtSyOgQAAMq8clWAAAAAAFeLq2B5ByNYAAAAAExDAQIAAADANIxgAQAAAJ5wMoPlDXRAAAAAAJiGAgQAAACAaRjBAgAAADzAVbC8gw4IAAAAANNQgAAAAAAwDSNYAAAAgCcYwfIKOiAAAAAATEMBAgAAAMA0jGABAAAAHrAVWR1B2UAHBAAAAIBpKEAAAAAAmIYRLAAAAMATXAXLK+iAAAAAADANBQgAAAAA0zCCBQAAAHjAxgiWV9ABAQAAAGAaChAAAAAApmEECwAAAPCEkxksb6ADAgAAAMA0FCAAAAAATMMIFgAAAOABroLlHXRAAAAAAJiGAgQAAACAaRjBAgAAADzBCJZX0AEBAAAAYBoKEAAAAACmYQQLAAAA8ABXwfIOOiAAAAAATEMBAgAAAMA0jGABAAAAnnAyg+UNdEAAAAAAmIYCBAAAAIBpGMECAAAAPMBVsLyDDggAAAAA01CAAAAAADANI1gAAACAJxjB8go6IAAAAABMQwECAAAAwDSMYAEAAAAe4CpY3kEHBAAAAIBpKEAAAAAAmIYRLAAAAMATRcxgeQMdEAAAAACmoQNikvY1IzQ0oocaBoToTN45LT2xVR/+9J8St727TmuNb97nou81cfen+vLnXZKkVV2f0XV+gW7b3PXNKzqVl+md4H0A+TUeOTbfrVH1NezeDmoYFqy0s9lavH635q7ZdtHtG9SuoaUTBritH0k6rd4TPzQwUt9Fjo1Ffo1HjuGLKEBM0Lx6Pb3R+hGtTfxRsw6uVasaDTQ0oocqyKa5P/3bbfuNKfs18LtZLms2m/TsjffLv5KfNqYekCQF2wN0nV+gpu77Uj+mHXfZPi0/y7Cfp7Qhv8Yjx+Zr0ShMU4f20lc7DmjmZxvVqkkdDb+3gyrYbJq9emuJ+zStW0uSFPvmp8orKChez8krKHH78o4cG4v8Go8cW4AJLK+gADFBbJPbdSAjURN+/FSStPmXg6pkq6A/NeqiBUc3KLfI9S99Wv45paWfc1l7sP6tahBQS4M2v6u0/POvNQ0KkyT9K3mvknLSjP9BSinyazxybL7Bd7fX/oRUPT9vtSRpU9wxVapQQQPubKP563YoN7/QbZ9m4SE6+Uu6dhxMMDtcn0SOjUV+jUeO4as4B8RglW0V1Tq4kf6VvNdlfV3SHvlX8lOrGg0u+x7X2QM0JKKHlhzfor3p//2F0TQwTBn52eX6H27k13jk2HyVK1XUzRHh+mbXIZf1r3cdlL/DrugmdUrcr1l4Le1PSDUjRJ9Hjo1Ffo1HjuHL6IAYrE7VYNkrVNLxrFMu6wm/Pq/nX1NbTh0qaddij0V0V5GzSO8eXOuy3jToep3Nz9brrR5Wm+saq4LNpg2p+zU1/kudyj3r3R+klCK/xiPH5guvWU32ypV0POWMy/qJ1DRJUv2QGtq877jbfk3Da+lI0inNG/OgmtUN0dmsXH2+OU6zPtukgqIiM0L3GeTYWOTXeOTYGtyI0DssLUAWL16se++9V3a7vXht8+bNmjNnjpKSkhQREaGhQ4eqSZMmFkZ5bQIrOyRJ5wpyXNazCvMkSf6V/C65fw27v35/fbQ+PrpBmb95j6aBYQpxBGl5wjb989hGNfCvpcER3fX3trHqt2m6cgrzvfiTlE7k13jk2HyBVc7nNDMnz2U969fn/lXsbvsEB1ZVzWr+cjqdmrZsgxLPZKhts3oacEeMatcI0Li5q40P3IeQY2ORX+ORY/gySwuQ559/Xl27dtV1110nSdqwYYNiY2PVoUMHdezYUXv27FHv3r01d+5ctW7d2spQr5pNtvN/uEjFXOS8dCl9X3gbVbDZtPDoJrfXXtqzWLmFBTpwNlGS9P2Zo/opM1kftB+iu69vrSUntlxT7L6A/BqPHJvPZruQ85JzW9KXlOdycjXk7SU6lnxayWfOXz1s58GTyi8o1PBeHTR71VYdSTptVMg+hxwbi/wajxzDl1l6DojzN39pZs6cqT/96U/64IMPNHbsWH300Ud66KGHNGXKFIsivHYXvvH97bfEVSue/2biXEHuJfe/PfQmbfnlUPFJu//rx7QTxf9wu2B32nGdzc9WRGDotYTtM8iv8cix+c5mn8+pv8P1G8yqvz7PzHHPeW5+obbGHy/+R8UF6/cckSRFhNc0IlSfRY6NRX6NR44t4nSWzoePKVUnoR87dky9evVyWXvwwQcVFxdnUUTXLiHrtAqKChVe9TqX9QvPf8pMuei+IX5BahZ0vb5O+tHttYBKDt1T52Y19A9xe61yhYrl5hKm5Nd45Nh8CalpKigsUt2Q6i7rdWudf/5T4im3ferXrqHenVq4/WPEUfl8ozstM8dtn/KMHBuL/BqPHMOXWVqAFLcPf9WgQQNlZbn+o+PMmTMKDHS/SZmvyCsq0Pdnjqpr6I0u691Cb1JGfrbi0k9cdN+o6nUlST+cOeb2Wn5RocZG3av+jTq7rHcJuUGOinbtOP2TF6Iv/civ8cix+fIKCrXrUIJub+V6/lv36AhlZOVo79Ekt31CqgfouT92U/foCJf1O2KaKjM7V/uOJxsas68hx8Yiv8Yjx/Bllp4D4nQ61a1bNzVs2FCNGzeW3W7X5MmTNX/+fFWuXFk7d+7UxIkT1aVLFyvDvGZzDv9LM9oM1Kut+uqzhB1qUb2e+jXspBn71yi3qED+Ff3UMCBECVmnXcZUmgTUVm5hvk5mu89j5hbl6x9H1mtQk9t1Ki9Tm1MPqklgqGKbdNOGlHhtO3XYzB/RUuTXeOTYfB+s2qpZI3vr9UF3a8V3e9WyUZj+1CNG05avV25+ofwddjUKC9aJ1HSlZWZrx4EEbdt/QqP7dJbDr7KOJp1Wp+YN9dBt0Xpr6bc6m3XpUbnyiBwbi/wajxybj6tgeYfN+dsTMUz0888/a//+/Tpw4EDx/z969Ki2b98uh8Oh6OhoNWvWTLNmzVKNGjWu+XhtVz/rhaivzm0hUYqN6Kb6/rWUmpOhT49v1oKjGyRJrYMb6t22sZr442J9eXJn8T5jo+5V19o36q5/vVrie9pkU+967dS7bjvVqRqs9PwsffXzD3rv0NduN4Yr68iv8cpDjgtW1DL9mJfStWVjDel5i+rXrqGU9HP65N/fa/668/m9OSJc749+QOM/XKPPN58fUw1w2DW45y26rWVj1azmr4TUdC34ZqeWbdxj5Y9RqpFjY5Ff45XFHO+cNcrqEC6q652vWx1Cif615imrQ7gilhYgJcnPz1flypUlSfv371fTpk3dRrWulpUFCIDSr7QVIABQHlGAXDlfK0BK1UnokoqLD0lq1qyZ14oPAAAA4Jo4S+njGhQVFWnatGnq1KmTWrZsqYEDB+rYMfdzN0vy+eefq1mzZkpISLiiY5a6AgQAAACAOWbOnKmFCxdq0qRJWrRokWw2m2JjY5WXl3fJ/U6ePKmJEyde1TEpQAAAAIByKC8vT3PmzNGIESPUpUsXRUZGaurUqUpOTtbatWsvul9RUZHGjBmjG2+88aLbXAoFCAAAAOABm9NZKh9XKz4+XufOnVP79u2L14KCghQVFaVt27ZddL93331X+fn5Gjx48FUd19LL8AIAAAC4Nt26dbvk6+vWrStxPSnp/P1iwsLCXNZDQkKUmJhY4j67d+/WnDlztHjxYiUnX929Y+iAAAAAAOVQdna2JMlut7us+/n5KTfX/b4wWVlZ+utf/6q//vWvatCgwVUflw4IAAAA4IkiqwMo2cU6HJfjcDgknT8X5MKfJSk3N1dVqlRx237SpElq0KCBHnrooasL9FcUIAAAAEA5dGH0KiUlRfXq1SteT0lJUWRkpNv2S5Yskd1uV3R0tCSpsLBQktSzZ0/de++9evHFFz06LgUIAAAAUA5FRkYqICBAW7ZsKS5AMjIyFBcXp379+rlt/9VXX7k8/+GHHzRmzBi99957aty4scfHpQABAAAAPHAtV5wqjex2u/r166cpU6YoODhYderU0eTJkxUaGqoePXqosLBQp0+fVmBgoBwOh+rXr++y/4WT2K+//npdd911Hh+Xk9ABAACAcmrkyJHq06ePxo0bp759+6pixYqaPXu27Ha7EhMT1bFjR61cudKrx7Q5nWWslLuEtquftToEAKVYwYpaVocAAOXezlmjrA7horrd/qrVIZRo3TfPWB3CFWEECwAAAPBEufna3liMYAEAAAAwDQUIAAAAANMwggUAAAB4ovycOm0oOiAAAAAATEMBAgAAAMA0jGABAAAAHrAxgeUVdEAAAAAAmIYCBAAAAIBpGMECAAAAPMFVsLyCDggAAAAA01CAAAAAADANI1gAAACAB2xFVkdQNtABAQAAAGAaChAAAAAApmEECwAAAPAEV8HyCjogAAAAAExDAQIAAADANIxgAQAAAJ5gAssr6IAAAAAAMA0FCAAAAADTMIIFAAAAeMDGVbC8gg4IAAAAANNQgAAAAAAwDSNYAAAAgCcYwfIKOiAAAAAATEMBAgAAAMA0jGABAAAAniiyOoCygQ4IAAAAANNQgAAAAAAwDSNYAAAAgAe4EaF30AEBAAAAYBoKEAAAAACmYQQLAAAA8AQjWF5BBwQAAACAaShAAAAAAJiGESwAAADAE4xgeUW5KkAKVtSyOgQAAACgXGMECwAAAIBpylUHBAAAALhqRVYHUDbQAQEAAABgGgoQAAAAAKZhBAsAAADwgI2rYHkFHRAAAAAApqEAAQAAAGAaRrAAAAAATzCC5RV0QAAAAACYhgIEAAAAgGkYwQIAAAA8wQiWV9ABAQAAAGAaChAAAAAApmEECwAAAPAEI1heQQcEAAAAgGkoQAAAAACYhhEsAAAAwBNFVgdQNtABAQAAAGAaChAAAAAApmEECwAAAPCAjatgeQUdEAAAAACmoQABAAAAYBpGsAAAAABPMILlFXRAAAAAAJiGAgQAAACAaRjBAgAAADxRxAiWN9ABAQAAAGAaChAAAAAApmEECwAAAPAEV8HyCjogAAAAAExDAQIAAADANIxgAQAAAJ5gBMsr6IAAAAAAMA0FCAAAAADTMIIFAAAAeIIRLK+gAwIAAADANBQgAAAAAEzDCBYAAADgiSJGsLyBDggAAAAA01CAAAAAADANI1gAAACAJ5xFVkdQJtABAQAAAGAaChAAAAAApmEECwAAAPAENyL0CjogAAAAAExDAQIAAADANIxgAQAAAJ7gRoReQQFikVuj6mvYvR3UMCxYaWeztXj9bs1ds+2i2zeoXUNLJwxwWz+SdFq9J35oYKS+ixwbi/wajxwbjxwbi/wajxzDF1GAWKBFozBNHdpLX+04oJmfbVSrJnU0/N4OqmCzafbqrSXu07RuLUlS7JufKq+goHg9J6+gxO3LO3JsLPJrPHJsPHJsLPJrPHIMX0UBYoHBd7fX/oRUPT9vtSRpU9wxVapQQQPubKP563YoN7/QbZ9m4SE6+Uu6dhxMMDtcn0SOjUV+jUeOjUeOjUV+jUeOLcBVsLyCk9BNVrlSRd0cEa5vdh1yWf9610H5O+yKblKnxP2ahdfS/oRUM0L0eeTYWOTXeOTYeOTYWOTXeOQYvowOiMnCa1aTvXIlHU8547J+IjVNklQ/pIY27zvutl/T8Fo6knRK88Y8qGZ1Q3Q2K1efb47TrM82qaCoyIzQfQY5Nhb5NR45Nh45Nhb5NR45hi+zvAD54YcftGXLFj322GOSpM2bN2vevHlKSEhQvXr1NHDgQMXExFgcpfcEVvGTJGXm5LmsZ/363L+K3W2f4MCqqlnNX06nU9OWbVDimQy1bVZPA+6IUe0aARo3d7XxgfsQcmws8ms8cmw8cmws8ms8cmwRRrC8wtICZPXq1Ro9erRuvfVWPfbYY/rXv/6lYcOGqXPnzurSpYsOHDig/v37a8aMGeratauVoXqNzWY7/4eLfIBL+vLhXE6uhry9RMeSTyv5TKYkaefBk8ovKNTwXh00e9VWHUk6bVTIPoccG4v8Go8cG48cG4v8Go8cw1uKioo0Y8YMffrpp8rIyNDNN9+s8ePHq379+iVuv2fPHk2ePFm7d++Wn5+f7rjjDv31r39VUFCQx8e09ByQGTNm6PHHH9cHH3wgSZo1a5aGDBmiv//97xozZozef/99Pf7445o2bZqVYXrV2excSZK/w/Wbiaq/Ps/MyXXbJze/UFvjjxf/srhg/Z4jkqSI8JpGhOqzyLGxyK/xyLHxyLGxyK/xyDG8ZebMmVq4cKEmTZqkRYsWyWazKTY2Vnl5eW7bpqSk6M9//rPq1aunZcuWaebMmdq5c6eeeuqpKzqmpQXI8ePHdc899xQ/T0hI0J133umyTc+ePXX48GGzQzNMQmqaCgqLVDekust63Vrnn/+UeMptn/q1a6h3pxZuv2Qclc83sNIycwyJ1VeRY2ORX+ORY+ORY2ORX+ORY4s4naXzcZXy8vI0Z84cjRgxQl26dFFkZKSmTp2q5ORkrV271m37kydPqlOnTho/frwaNGig1q1b64EHHtB33313Rce1tACpW7eu/vOf/xQ/v+GGGxQfH++yze7du1W7dm2zQzNMXkGhdh1K0O2tmrisd4+OUEZWjvYeTXLbJ6R6gJ77Yzd1j45wWb8jpqkys3O173iyoTH7GnJsLPJrPHJsPHJsLPJrPHIMb4iPj9e5c+fUvn374rWgoCBFRUVp2zb3G1pGR0frzTffVKVK54vWQ4cOadmyZerQocMVHdfSc0BiY2P13HPPKSkpST179tSwYcP09NNPKzc3VxEREfrhhx/0zjvv6PHHH7cyTK/7YNVWzRrZW68Pulsrvturlo3C9KceMZq2fL1y8wvl77CrUViwTqSmKy0zWzsOJGjb/hMa3aezHH6VdTTptDo1b6iHbovWW0u/1dks9zZreUeOjUV+jUeOjUeOjUV+jUeOcUG3bt0u+fq6detKXE9KOl+ohoWFuayHhIQoMTHxku9555136ujRo6pTp45mzpx5BdFKNqfT2tP5V6xYoWnTpunkyZOy2Wz633D8/f01aNAgDR061CvHaj10qlfexxu6tmysIT1vUf3aNZSSfk6f/Pt7zV+3U5J0c0S43h/9gMZ/uEafb46TJAU47Brc8xbd1rKxalbzV0JquhZ8s1PLNu6x8sco1cixsciv8cix8cixsciv8cpijnfOGmV1CBd1V9hwq0MoUV5U/CVfv1gBsmLFCo0dO1b79u1ThQr/HYwaO3asUlJSNG/evIu+548//qicnBxNmTJFp06d0ooVK+Tv7+9RvJYXIBf89NNPOnr0qDIzM1W5cmWFhoYqKipKfn5+XjtGaSpAAAAA4I4C5MqtSnznqvZbs2aNRo4cqR9++EEOh6N4/S9/+Yvy8vI0a9asy75HSkqKunTpoldffVX33XefR8e1/D4gFzRq1EiNGjWyOgwAAACgXLgwepWSkqJ69eoVr6ekpCgyMtJt+8OHDyshIUFdunQpXgsJCVG1atWUnOz5OUSWnoQOAAAA+Ayrr3bl5atgRUZGKiAgQFu2bCley8jIUFxcXIk3Al+/fr3+8pe/KDPzv5dyPn78uM6cOaPGjRt7fFwKEAAAAKAcstvt6tevn6ZMmaJ169YpPj5eo0aNUmhoqHr06KHCwkKlpqYqJ+f8JZp79eqlwMBAjRkzRgcPHtT27ds1cuRItWjR4opuGk4BAgAAAJRTI0eOVJ8+fTRu3Dj17dtXFStW1OzZs2W325WYmKiOHTtq5cqVkqQaNWroo48+UlFRkfr27avhw4crKipKs2fPVsWKFT0+Zqk5Cd0MnIQOAABQupXqk9Bre+fKrN62KvnyJ4uXJnRAAAAAAJiGAgQAAACAaUrNZXgBAACAUq2o3Jy5YCg6IAAAAABMQwECAAAAwDSMYAEAAAAecDqLrA6hTKADAgAAAMA0FCAAAAAATMMIFgAAAOAJroLlFXRAAAAAAJiGAgQAAACAaRjBAgAAADzhZATLG+iAAAAAADANBQgAAAAA0zCCBQAAAHiiiBsRegMdEAAAAACmoQABAAAAYBpGsAAAAABPcBUsr6ADAgAAAMA0FCAAAAAATMMIFgAAAOABJ1fB8go6IAAAAABMQwECAAAAwDSMYAEAAACe4CpYXkEHBAAAAIBpKEAAAAAAmIYRLAAAAMATRYxgeQMdEAAAAACmoQABAAAAYBpGsAAAAABPOLkRoTfQAQEAAABgGgoQAAAAAKZhBAsAAADwgJOrYHkFHRAAAAAApqEAAQAAAGAaRrAAAAAAT3AVLK+gAwIAAADANBQgAAAAAEzDCBYAAADgAa6C5R10QAAAAACYhgIEAAAAgGkYwQIAAAA8wVWwvIIOCAAAAADTUIAAAAAAMI3N6XRyOj8AAAAAU9ABAQAAAGAaChAAAAAApqEAAQAAAGAaChAAAAAApqEAAQAAAGAaChAAAAAApqEAAQAAAGAaChAAAAAApqEAAQAAAGAaChAAAAAApqEAAQAAAGAaChAAAAAApqEAAQAAAGAaCpBSpqioSNOmTVOnTp3UsmVLDRw4UMeOHbM6rDJr5syZeuSRR6wOo0xJS0vTCy+8oM6dO6t169bq27evtm/fbnVYZcqpU6c0ZswYtW/fXtHR0Xrsscd06NAhq8Mqk44cOaLo6GgtXbrU6lDKlJMnT6pZs2Zuj08//dTq0MqU5cuX6/e//72aN2+uu+++W6tWrbI6JEASBUipM3PmTC1cuFCTJk3SokWLZLPZFBsbq7y8PKtDK3PmzZunadOmWR1GmTN69Gj98MMPevPNN7V48WLdeOONevTRR3X48GGrQyszhg4dqhMnTuj999/X4sWL5XA4NGDAAGVnZ1sdWpmSn5+vv/71r8rKyrI6lDJn//798vPz0/r167Vhw4bixz333GN1aGXGihUr9Oyzz+rBBx/UF198od///vcaPXq0du3aZXVoAAVIaZKXl6c5c+ZoxIgR6tKliyIjIzV16lQlJydr7dq1VodXZiQnJ2vQoEF6++231bBhQ6vDKVOOHTumjRs3avz48YqJiVGjRo303HPPqXbt2vriiy+sDq9MOHPmjMLDw/XSSy+pefPmaty4sYYNG6bU1FQdPHjQ6vDKlOnTp8vf39/qMMqkAwcOqGHDhgoJCVGtWrWKHw6Hw+rQygSn06m3335b/fv3V//+/VW/fn0NHz5ct956q7Zu3Wp1eAAFSGkSHx+vc+fOqX379sVrQUFBioqK0rZt2yyMrGzZu3evqlWrps8++0wtW7a0OpwypUaNGnrvvfd00003Fa/ZbDY5nU6lp6dbGFnZUaNGDb355puKiIiQJP3yyy+aPXu2QkND1aRJE4ujKzu2bdumRYsW6fXXX7c6lDJp//79fF4N9NNPP+nkyZNuHaXZs2dr8ODBFkUF/FclqwPAfyUlJUmSwsLCXNZDQkKUmJhoRUhl0u23367bb7/d6jDKpKCgIHXp0sVlbdWqVTp+/Lg6duxoUVRl1/PPP69PPvlEdrtds2bNUtWqVa0OqUzIyMjQ2LFjNW7cOLffx/COAwcOqFatWvrjH/+oo0ePqn79+ho2bJg6depkdWhlwtGjRyVJWVlZevTRRxUXF6fw8HANHTqU//6hVKADUopcmN+22+0u635+fsrNzbUiJOCa7NixQ88++6y6devGf/QM0L9/fy1ZskT33nuvhg8frr1791odUpkwYcIEtWrVivMRDJKXl6ejR48qMzNTTzzxhN577z01b95csbGx+u6776wOr0zIzMyUJD311FPq2bOn5syZow4dOmjYsGHkGKUCHZBS5MLsa15ensscbG5urqpUqWJVWMBV+frrr/XXv/5VLVu21Jtvvml1OGXShRGWl156Sd9//73mz5+vV1991eKofNvy5cu1fft2ff7551aHUmbZ7XZt27ZNlSpVKv7C7aabbtLhw4c1e/Zs3XLLLRZH6PsqV64sSXr00Uf1hz/8QZJ0ww03KC4uTnPnziXHsBwdkFLkQqs/JSXFZT0lJUWhoaFWhARclfnz52vEiBHq3Lmz3n//fU4s9aJTp07piy++UGFhYfFahQoV1LhxY7ffHbhyS5Ys0alTp3TbbbcpOjpa0dHRkqTx48fr7rvvtji6sqNq1apu3f6mTZsqOTnZoojKlgv/ZmjatKnLepMmTZSQkGBFSIALCpBSJDIyUgEBAdqyZUvxWkZGhuLi4hQTE2NhZIDnFixYoJdeekkPP/yw3nrrLbd/ZODapKSk6Mknn3S5kk1+fr7i4uLUuHFjCyMrG6ZMmaKVK1dq+fLlxQ9JGjlypN577z1rgysj4uPjFR0d7XZ/oD179nBiupdERUXJ399fP/zwg8v6gQMHVK9ePYuiAv6LEaxSxG63q1+/fpoyZYqCg4NVp04dTZ48WaGhoerRo4fV4QGXdeTIEb3yyivq0aOHBg8erFOnThW/5nA4FBgYaGF0ZUNkZKQ6duyoiRMnatKkSQoKCtK7776rjIwMDRgwwOrwfF7t2rVLXL/uuutUp04dk6Mpm5o2baqIiAhNnDhR48ePV40aNfTJJ5/o+++/1+LFi60Or0xwOBwaNGiQ3nnnHdWuXVstWrTQl19+qY0bN2revHlWhwdQgJQ2I0eOVEFBgcaNG6ecnBy1adNGs2fP5ltk+IQ1a9YoPz9fa9eudbt3zR/+8Ae99tprFkVWdthsNr311lt644039MQTT+js2bOKiYnRxx9/rOuvv97q8IDLqlChgt59911NmTJFTzzxhDIyMhQVFaW5c+eqWbNmVodXZgwbNkxVqlQpvp9Y48aNNX36dLVr187q0ADZnE6n0+ogAAAAAJQPnAMCAAAAwDQUIAAAAABMQwECAAAAwDQUIAAAAABMQwECAAAAwDQUIAAAAABMQwECAAAAwDQUIAAAAABMQwECAD4mNzdXUVFRio6O1ksvvWR1OAAAXBEKEADwMTabTR9++KFatGih+fPn68iRI1aHBACAxyhAAMDH2O12tWnTRoMGDZIk7d271+KIAADwHAUIAPioRo0aSZL27dtncSQAAHiOAgQAfNT7778vSYqPj7c4EgAAPEcBAgA+aMOGDfrnP/+patWqKS4uzupwAADwGAUIAPiYjIwMPfvss+rWrZv69u2r06dPKzk52eqwAADwCAUIAPiYiRMnqqCgQJMmTVJUVJQkxrAAAL6DAgQAfMjq1av1xRdf6OWXX1ZwcHBxAcKJ6AAAX0EBAgA+IjU1VePHj9eDDz6orl27SpLq1q2roKAgzgMBAPgMChAA8BHPP/+8qlWrpqefftpl/YYbbmAECwDgMyhAAMAHfPrpp/r222/1t7/9TVWrVnV5LSoqSsePH1dmZqZF0QEA4Dmb0+l0Wh0EAAAAgPKBDggAAAAA01CAAAAAADANBQgAAAAA01CAAAAAADANBQgAAAAA01CAAAAAADANBQgAAAAA01CAAAAAADANBQgAAAAA01CAAAAAADANBQgAAAAA01CAAAAAADDN/wOpUE87FNLINAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# import necessary packages\n", "import numpy as np\n", @@ -407,9 +630,7 @@ { "cell_type": "markdown", "id": "1c0ad443", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "How do we interpret these results?" ] @@ -417,9 +638,7 @@ { "cell_type": "markdown", "id": "1cea859c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Lecture Thursday October 26" ] @@ -427,9 +646,7 @@ { "cell_type": "markdown", "id": "bbdb3e49", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Developing a code for doing neural networks with back propagation\n", "\n", @@ -456,9 +673,7 @@ { "cell_type": "markdown", "id": "45d25695", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Collect and pre-process data\n", "\n", @@ -506,10 +721,7 @@ "cell_type": "code", "execution_count": 3, "id": "87c0ee64", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# import necessary packages\n", @@ -559,9 +771,7 @@ { "cell_type": "markdown", "id": "4e780f35", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Train and test datasets\n", "\n", @@ -580,10 +790,7 @@ "cell_type": "code", "execution_count": 4, "id": "bb737921", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.model_selection import train_test_split\n", @@ -618,9 +825,7 @@ { "cell_type": "markdown", "id": "ffb0a087", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Define model and architecture\n", "\n", @@ -662,9 +867,7 @@ { "cell_type": "markdown", "id": "fed0ab83", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Layers\n", "\n", @@ -702,9 +905,7 @@ { "cell_type": "markdown", "id": "6f3ab464", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Weights and biases\n", "\n", @@ -723,10 +924,7 @@ "cell_type": "code", "execution_count": 5, "id": "6745d89e", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# building our neural network\n", @@ -749,9 +947,7 @@ { "cell_type": "markdown", "id": "c06d038e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Feed-forward pass\n", "\n", @@ -777,9 +973,7 @@ { "cell_type": "markdown", "id": "4f0e93ce", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Matrix multiplications\n", "\n", @@ -814,10 +1008,7 @@ "cell_type": "code", "execution_count": 6, "id": "3c39783f", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# setup the feed-forward pass, subscript h = hidden layer\n", @@ -860,9 +1051,7 @@ { "cell_type": "markdown", "id": "c956d7fe", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Choose cost function and optimizer\n", "\n", @@ -891,9 +1080,7 @@ { "cell_type": "markdown", "id": "e939a07d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Optimizing the cost function\n", "\n", @@ -929,9 +1116,7 @@ { "cell_type": "markdown", "id": "ccdee2fd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Regularization\n", "\n", @@ -963,9 +1148,7 @@ { "cell_type": "markdown", "id": "1bf8ea1a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Matrix multiplication\n", "\n", @@ -1004,10 +1187,7 @@ "cell_type": "code", "execution_count": 7, "id": "addc476d", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# to categorical turns our integer vector into a onehot representation\n", @@ -1083,9 +1263,7 @@ { "cell_type": "markdown", "id": "435d5f8e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Improving performance\n", "\n", @@ -1104,9 +1282,7 @@ { "cell_type": "markdown", "id": "4d67793b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Full object-oriented implementation\n", "\n", @@ -1118,10 +1294,7 @@ "cell_type": "code", "execution_count": 8, "id": "e189ad94", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "class NeuralNetwork:\n", @@ -1228,9 +1401,7 @@ { "cell_type": "markdown", "id": "131559a9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Evaluate model performance on test data\n", "\n", @@ -1247,10 +1418,7 @@ "cell_type": "code", "execution_count": 9, "id": "6014b15e", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "epochs = 100\n", @@ -1274,9 +1442,7 @@ { "cell_type": "markdown", "id": "a215be5c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Adjust hyperparameters\n", "\n", @@ -1288,10 +1454,7 @@ "cell_type": "code", "execution_count": 10, "id": "3c371733", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "eta_vals = np.logspace(-5, 1, 7)\n", @@ -1319,9 +1482,7 @@ { "cell_type": "markdown", "id": "ed44fd1b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Visualization" ] @@ -1330,10 +1491,7 @@ "cell_type": "code", "execution_count": 11, "id": "38c2057e", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# visual representation of grid search\n", @@ -1374,9 +1532,7 @@ { "cell_type": "markdown", "id": "647e5296", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## scikit-learn implementation\n", "\n", @@ -1397,10 +1553,7 @@ "cell_type": "code", "execution_count": 12, "id": "277402f6", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.neural_network import MLPClassifier\n", @@ -1424,9 +1577,7 @@ { "cell_type": "markdown", "id": "cfbf38a6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Visualization" ] @@ -1435,10 +1586,7 @@ "cell_type": "code", "execution_count": 13, "id": "920c188a", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# optional\n", @@ -1480,9 +1628,7 @@ { "cell_type": "markdown", "id": "97323847", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Building neural networks in Tensorflow and Keras\n", "\n", @@ -1498,9 +1644,7 @@ { "cell_type": "markdown", "id": "c2a7c615", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Tensorflow\n", "\n", @@ -1533,10 +1677,7 @@ "cell_type": "code", "execution_count": 14, "id": "bba703cc", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "pip3 install tensorflow" @@ -1545,9 +1686,7 @@ { "cell_type": "markdown", "id": "a4068463", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and/or if you use **anaconda**, just write (or install from the graphical user interface)\n", "(current release of CPU-only TensorFlow)" @@ -1557,10 +1696,7 @@ "cell_type": "code", "execution_count": 15, "id": "d6bd0dd0", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "conda create -n tf tensorflow\n", @@ -1570,9 +1706,7 @@ { "cell_type": "markdown", "id": "acc51b3f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "To install the current release of GPU TensorFlow" ] @@ -1581,10 +1715,7 @@ "cell_type": "code", "execution_count": 16, "id": "a0672dbc", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "conda create -n tf-gpu tensorflow-gpu\n", @@ -1594,9 +1725,7 @@ { "cell_type": "markdown", "id": "63f07738", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Using Keras\n", "\n", @@ -1609,10 +1738,7 @@ "cell_type": "code", "execution_count": 17, "id": "9a9dc1f6", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "conda install keras" @@ -1621,9 +1747,7 @@ { "cell_type": "markdown", "id": "69b5c338", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "You can look up the [instructions here](https://keras.io/) for more information.\n", "\n", @@ -1633,9 +1757,7 @@ { "cell_type": "markdown", "id": "4a78e3e1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Collect and pre-process data\n", "\n", @@ -1646,10 +1768,7 @@ "cell_type": "code", "execution_count": 18, "id": "08e4f938", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# import necessary packages\n", @@ -1701,10 +1820,7 @@ "cell_type": "code", "execution_count": 19, "id": "07f3f725", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from tensorflow.keras.layers import Input\n", @@ -1730,10 +1846,7 @@ "cell_type": "code", "execution_count": 20, "id": "646f3c64", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -1760,10 +1873,7 @@ "cell_type": "code", "execution_count": 21, "id": "02e48d62", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "DNN_keras = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)\n", @@ -1787,10 +1897,7 @@ "cell_type": "code", "execution_count": 22, "id": "b870683f", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# optional\n", @@ -1829,9 +1936,7 @@ { "cell_type": "markdown", "id": "854ae0f4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The Breast Cancer Data, now with Keras" ] @@ -1840,10 +1945,7 @@ "cell_type": "code", "execution_count": 23, "id": "4ccfdbf2", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -2017,9 +2119,7 @@ { "cell_type": "markdown", "id": "9b5849c3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Fine-tuning neural network hyperparameters\n", "\n", @@ -2045,9 +2145,7 @@ { "cell_type": "markdown", "id": "d5e48e8e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Hidden layers\n", "\n", @@ -2068,9 +2166,7 @@ { "cell_type": "markdown", "id": "58942375", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Which activation function should I use?\n", "\n", @@ -2099,9 +2195,7 @@ { "cell_type": "markdown", "id": "04de1245", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Is the Logistic activation function (Sigmoid) our choice?\n", "\n", @@ -2131,9 +2225,7 @@ { "cell_type": "markdown", "id": "a00515d3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The derivative of the Logistic funtion\n", "\n", @@ -2169,9 +2261,7 @@ { "cell_type": "markdown", "id": "a31e0434", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The RELU function family\n", "\n", @@ -2194,9 +2284,7 @@ { "cell_type": "markdown", "id": "200106bb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "ELU(z) = \\left\\{\\begin{array}{cc} \\alpha\\left( \\exp{(z)}-1\\right) & z < 0,\\\\ z & z \\ge 0.\\end{array}\\right.\n", @@ -2206,9 +2294,7 @@ { "cell_type": "markdown", "id": "ec25493e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Which activation function should we use?\n", "\n", @@ -2229,9 +2315,7 @@ { "cell_type": "markdown", "id": "3c158df0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## More on activation functions, output layers\n", "\n", @@ -2249,9 +2333,7 @@ { "cell_type": "markdown", "id": "f3320b0b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Batch Normalization\n", "\n", @@ -2271,9 +2353,7 @@ { "cell_type": "markdown", "id": "05cf40b3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Dropout\n", "\n", @@ -2289,9 +2369,7 @@ { "cell_type": "markdown", "id": "0d29278d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Gradient Clipping\n", "\n", @@ -2308,9 +2386,7 @@ { "cell_type": "markdown", "id": "55e322d2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## A very nice website on Neural Networks\n", "\n", @@ -2320,9 +2396,7 @@ { "cell_type": "markdown", "id": "c5f566a8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## A top-down perspective on Neural networks\n", "\n", @@ -2365,9 +2439,7 @@ { "cell_type": "markdown", "id": "baa5a3ba", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Limitations of supervised learning with deep networks\n", "\n", @@ -2395,9 +2467,7 @@ { "cell_type": "markdown", "id": "9398e7b0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Solving ODEs with Deep Learning\n", "\n", @@ -2422,9 +2492,7 @@ { "cell_type": "markdown", "id": "38ca5c29", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Ordinary Differential Equations\n", "\n", @@ -2436,9 +2504,7 @@ { "cell_type": "markdown", "id": "0c60c696", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -2453,9 +2519,7 @@ { "cell_type": "markdown", "id": "f2583e1e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $g(x)$ is the function to find, and $g^{(n)}(x)$ is the $n$-th derivative of $g(x)$.\n", "\n", @@ -2469,9 +2533,7 @@ { "cell_type": "markdown", "id": "39729d0e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The trial solution\n", "\n", @@ -2481,9 +2543,7 @@ { "cell_type": "markdown", "id": "161e15ef", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -2499,9 +2559,7 @@ { "cell_type": "markdown", "id": "f9cf300b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $h_1(x)$ is a function that makes $g_t(x)$ satisfy a given set\n", "of conditions, $N(x,P)$ a neural network with weights and biases\n", @@ -2520,9 +2578,7 @@ { "cell_type": "markdown", "id": "733ed455", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Minimization process\n", "\n", @@ -2537,9 +2593,7 @@ { "cell_type": "markdown", "id": "f53e9f95", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C\\left(x, P\\right) = \\big(f\\left(x, \\, g(x), \\, g'(x), \\, g''(x), \\, \\dots \\, , \\, g^{(n)}(x)\\right)\\big)^2\n", @@ -2549,9 +2603,7 @@ { "cell_type": "markdown", "id": "34517bb4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "If $N$ inputs are given as a vector $\\boldsymbol{x}$ with elements $x_i$ for $i = 1,\\dots,N$,\n", "the cost function becomes" @@ -2560,9 +2612,7 @@ { "cell_type": "markdown", "id": "e02c971a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -2577,9 +2627,7 @@ { "cell_type": "markdown", "id": "8556f292", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The neural net should then find the parameters $P$ that minimizes the cost function in\n", "([3](#cost)) for a set of $N$ training samples $x_i$." @@ -2588,9 +2636,7 @@ { "cell_type": "markdown", "id": "cca6ee92", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Minimizing the cost function using gradient descent and automatic differentiation\n", "\n", @@ -2604,9 +2650,7 @@ { "cell_type": "markdown", "id": "5c2b6a47", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example: Exponential decay\n", "\n", @@ -2616,9 +2660,7 @@ { "cell_type": "markdown", "id": "10ceefea", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -2633,9 +2675,7 @@ { "cell_type": "markdown", "id": "7a6beee0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $g(0) = g_0$ for some chosen initial value $g_0$.\n", "\n", @@ -2645,9 +2685,7 @@ { "cell_type": "markdown", "id": "53982347", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -2663,9 +2701,7 @@ { "cell_type": "markdown", "id": "ce561b16", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Having an analytical solution at hand, it is possible to use it to compare how well a neural network finds a solution of ([4](#solve_expdec))." ] @@ -2673,9 +2709,7 @@ { "cell_type": "markdown", "id": "b5d5314b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The function to solve for\n", "\n", @@ -2685,9 +2719,7 @@ { "cell_type": "markdown", "id": "c4d67b46", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -2702,9 +2734,7 @@ { "cell_type": "markdown", "id": "fe827114", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $g(0) = g_0$ with $\\gamma$ and $g_0$ being some chosen values.\n", "\n", @@ -2714,9 +2744,7 @@ { "cell_type": "markdown", "id": "8ffca9fa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The trial solution\n", "To begin with, a trial solution $g_t(t)$ must be chosen. A general trial solution for ordinary differential equations could be" @@ -2725,9 +2753,7 @@ { "cell_type": "markdown", "id": "9ec1e42a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "g_t(x, P) = h_1(x) + h_2(x, N(x, P))\n", @@ -2737,9 +2763,7 @@ { "cell_type": "markdown", "id": "a002d557", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $h_1(x)$ ensuring that $g_t(x)$ satisfies some conditions and $h_2(x,N(x, P))$ an expression involving $x$ and the output from the neural network $N(x,P)$ with $P $ being the collection of the weights and biases for each layer. For now, it is assumed that the network consists of one input layer, one hidden layer, and one output layer." ] @@ -2747,9 +2771,7 @@ { "cell_type": "markdown", "id": "760f494e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Setup of Network\n", "\n", @@ -2767,9 +2789,7 @@ { "cell_type": "markdown", "id": "5f15f368", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -2784,9 +2804,7 @@ { "cell_type": "markdown", "id": "c91d6d7c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Reformulating the problem\n", "\n", @@ -2803,9 +2821,7 @@ { "cell_type": "markdown", "id": "bd6ed5a3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "g_t(x, P) = g_0 + x \\cdot N(x, P)\n", @@ -2815,9 +2831,7 @@ { "cell_type": "markdown", "id": "06a70c9a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "has been chosen such that it already solves the condition $g(0) = g_0$. What remains, is to find $P$ such that" ] @@ -2825,9 +2839,7 @@ { "cell_type": "markdown", "id": "6d20f780", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -2842,9 +2854,7 @@ { "cell_type": "markdown", "id": "9546abd9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "is fulfilled as *best as possible*." ] @@ -2852,9 +2862,7 @@ { "cell_type": "markdown", "id": "e4073ebd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## More technicalities\n", "\n", @@ -2868,9 +2876,7 @@ { "cell_type": "markdown", "id": "2dfb903c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\min_{P}\\Big\\{ \\big(g_t'(x, P) - ( -\\gamma g_t(x, P) \\big)^2 \\Big\\}\n", @@ -2880,9 +2886,7 @@ { "cell_type": "markdown", "id": "b1890129", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "(the notation $\\min_{P}\\{ f(x, P) \\}$ means that we desire to find $P$ that yields the minimum of $f(x, P)$)\n", "\n", @@ -2892,9 +2896,7 @@ { "cell_type": "markdown", "id": "a787ccbe", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\min_{P_{\\text{hidden} }, \\ P_{\\text{output} }}\\Big\\{ \\big(g_t'(x, \\{ P_{\\text{hidden} }, P_{\\text{output} }\\}) - ( -\\gamma g_t(x, \\{ P_{\\text{hidden} }, P_{\\text{output} }\\}) \\big)^2 \\Big\\}\n", @@ -2904,9 +2906,7 @@ { "cell_type": "markdown", "id": "f0a4fa04", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "for an input value $x$." ] @@ -2914,9 +2914,7 @@ { "cell_type": "markdown", "id": "94e4bd68", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## More details\n", "\n", @@ -2926,9 +2924,7 @@ { "cell_type": "markdown", "id": "1c418fe5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -2943,9 +2939,7 @@ { "cell_type": "markdown", "id": "92fee060", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Letting $\\boldsymbol{x}$ be a vector with elements $x_i$ and $C(\\boldsymbol{x}, P) = \\frac{1}{N} \\sum_i \\big(g_t'(x_i, P) - ( -\\gamma g_t(x_i, P) \\big)^2$ denote the cost function, the minimization problem that our network must solve, becomes" ] @@ -2953,9 +2947,7 @@ { "cell_type": "markdown", "id": "81b76e58", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\min_{P} C(\\boldsymbol{x}, P)\n", @@ -2965,9 +2957,7 @@ { "cell_type": "markdown", "id": "2d3f26a5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In terms of $P_{\\text{hidden} }$ and $P_{\\text{output} }$, this could also be expressed as\n", "\n", @@ -2979,9 +2969,7 @@ { "cell_type": "markdown", "id": "872160e6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## A possible implementation of a neural network\n", "\n", @@ -2995,9 +2983,7 @@ { "cell_type": "markdown", "id": "f7c201c9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Technicalities\n", "\n", @@ -3007,9 +2993,7 @@ { "cell_type": "markdown", "id": "67b3b2c5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{aligned}\n", @@ -3029,9 +3013,7 @@ { "cell_type": "markdown", "id": "4b0c1a9f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Final technicalities I\n", "\n", @@ -3041,9 +3023,7 @@ { "cell_type": "markdown", "id": "7d711054", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{aligned}\n", @@ -3064,9 +3044,7 @@ { "cell_type": "markdown", "id": "959f1555", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Final technicalities II\n", "\n", @@ -3080,9 +3058,7 @@ { "cell_type": "markdown", "id": "0ee5702b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "f(z) = \\frac{1}{1 + \\exp{(-z)}}\n", @@ -3092,9 +3068,7 @@ { "cell_type": "markdown", "id": "86386ed5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "It is possible to use other activations functions for the hidden layer also.\n", "\n", @@ -3116,9 +3090,7 @@ { "cell_type": "markdown", "id": "792a7767", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Final technicalities III\n", "\n", @@ -3128,9 +3100,7 @@ { "cell_type": "markdown", "id": "debfdf7d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{aligned}\n", @@ -3149,9 +3119,7 @@ { "cell_type": "markdown", "id": "cec57344", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Final technicalities IV\n", "\n", @@ -3161,9 +3129,7 @@ { "cell_type": "markdown", "id": "fc406bc4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{z}_{1}^{\\text{output}} =\n", @@ -3180,9 +3146,7 @@ { "cell_type": "markdown", "id": "8280357c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In this case we seek a continuous range of values since we are approximating a function. This means that after computing $\\boldsymbol{z}_{1}^{\\text{output}}$ the neural network has finished its feed forward step, and $\\boldsymbol{z}_{1}^{\\text{output}}$ is the final output of the network." ] @@ -3190,9 +3154,7 @@ { "cell_type": "markdown", "id": "e682cf5d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Back propagation\n", "\n", @@ -3204,9 +3166,7 @@ { "cell_type": "markdown", "id": "86c630c3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{x}, P) = \\frac{1}{N} \\sum_i \\big(g_t'(x_i, P) - ( -\\gamma g_t(x_i, P) \\big)^2\n", @@ -3216,9 +3176,7 @@ { "cell_type": "markdown", "id": "64bc4c3a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In order to minimize the cost function, an optimization method must be chosen.\n", "\n", @@ -3228,9 +3186,7 @@ { "cell_type": "markdown", "id": "329b6929", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Gradient descent\n", "\n", @@ -3245,9 +3201,7 @@ { "cell_type": "markdown", "id": "ffdb4b35", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\omega}_{\\text{new} } = \\boldsymbol{\\omega} - \\lambda \\nabla_{\\boldsymbol{\\omega}} C(\\boldsymbol{x}, \\boldsymbol{\\omega})\n", @@ -3257,9 +3211,7 @@ { "cell_type": "markdown", "id": "972f3fe7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "for a number of iterations or until $ \\big|\\big| \\boldsymbol{\\omega}_{\\text{new} } - \\boldsymbol{\\omega} \\big|\\big|$ becomes smaller than some given tolerance.\n", "\n", @@ -3279,9 +3231,7 @@ { "cell_type": "markdown", "id": "26f47c23", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{aligned}\n", @@ -3294,9 +3244,7 @@ { "cell_type": "markdown", "id": "46e2f864", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The code for solving the ODE" ] @@ -3305,10 +3253,7 @@ "cell_type": "code", "execution_count": 24, "id": "5c9ef3e2", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -3460,9 +3405,7 @@ { "cell_type": "markdown", "id": "9541ca3b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The network with one input layer, specified number of hidden layers, and one output layer\n", "\n", @@ -3475,10 +3418,7 @@ "cell_type": "code", "execution_count": 25, "id": "cb1f580a", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -3644,9 +3584,7 @@ { "cell_type": "markdown", "id": "1ce3dba0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example: Population growth\n", "\n", @@ -3657,9 +3595,7 @@ { "cell_type": "markdown", "id": "bd8d79a2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -3674,9 +3610,7 @@ { "cell_type": "markdown", "id": "3a2583e3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $g(t)$ is the population density at time $t$, $\\alpha > 0$ the growth rate and $A > 0$ is the maximum population number in the environment.\n", "Also, at $t = 0$ the population has the size $g(0) = g_0$, where $g_0$ is some chosen constant.\n", @@ -3690,9 +3624,7 @@ { "cell_type": "markdown", "id": "bda6ba67", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Setting up the problem\n", "\n", @@ -3703,9 +3635,7 @@ { "cell_type": "markdown", "id": "1483f8e7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -3720,9 +3650,7 @@ { "cell_type": "markdown", "id": "82aa5d57", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $g(0) = g_0$.\n", "\n", @@ -3732,9 +3660,7 @@ { "cell_type": "markdown", "id": "ba1d1722", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The trial solution\n", "\n", @@ -3759,9 +3685,7 @@ { "cell_type": "markdown", "id": "58ccc1eb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The program using Autograd\n", "\n", @@ -3772,10 +3696,7 @@ "cell_type": "code", "execution_count": 26, "id": "0a4c20d1", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -3946,9 +3867,7 @@ { "cell_type": "markdown", "id": "97450de8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Using forward Euler to solve the ODE\n", "\n", @@ -3966,9 +3885,7 @@ { "cell_type": "markdown", "id": "89c1f1e5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{aligned}\n", @@ -3981,9 +3898,7 @@ { "cell_type": "markdown", "id": "fa3644af", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "along with the condition that $g(0) = g_0$.\n", "\n", @@ -3995,9 +3910,7 @@ { "cell_type": "markdown", "id": "107bef62", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{aligned}\n", @@ -4011,9 +3924,7 @@ { "cell_type": "markdown", "id": "65796f1d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Now, if $g_i = g(t_i)$ then" ] @@ -4021,9 +3932,7 @@ { "cell_type": "markdown", "id": "fdb684ca", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -4043,9 +3952,7 @@ { "cell_type": "markdown", "id": "405eb53d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "for $i \\geq 1$ and $g_0 = g(t_0) = g(0) = g_0$.\n", "\n", @@ -4057,10 +3964,7 @@ "cell_type": "code", "execution_count": 27, "id": "a903f1cf", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# Assume that all function definitions from the example program using Autograd\n", @@ -4133,9 +4037,7 @@ { "cell_type": "markdown", "id": "f79a730a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example: Solving the one dimensional Poisson equation\n", "\n", @@ -4145,9 +4047,7 @@ { "cell_type": "markdown", "id": "28015c61", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -4162,9 +4062,7 @@ { "cell_type": "markdown", "id": "54ab04bb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $f(x)$ is a given function for $x \\in (0,1)$.\n", "\n", @@ -4174,9 +4072,7 @@ { "cell_type": "markdown", "id": "9cc20be6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{align*}\n", @@ -4189,9 +4085,7 @@ { "cell_type": "markdown", "id": "7af7511b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "This equation can be solved numerically using programs where e.g Autograd and TensorFlow are used.\n", "The results from the networks can then be compared to the analytical solution.\n", @@ -4201,9 +4095,7 @@ { "cell_type": "markdown", "id": "96856fc2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The specific equation to solve for\n", "\n", @@ -4213,9 +4105,7 @@ { "cell_type": "markdown", "id": "1cf407bd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "-g''(x) = f(x),\\qquad x \\in (0,1)\n", @@ -4225,9 +4115,7 @@ { "cell_type": "markdown", "id": "493a79a2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $f(x)$ is a given function, along with the chosen conditions" ] @@ -4235,9 +4123,7 @@ { "cell_type": "markdown", "id": "8b735b17", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -4252,9 +4138,7 @@ { "cell_type": "markdown", "id": "39ddedfe", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In this example, we consider the case when $f(x) = (3x + x^2)\\exp(x)$.\n", "\n", @@ -4264,9 +4148,7 @@ { "cell_type": "markdown", "id": "46bba0f0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "g_t(x) = x \\cdot (1-x) \\cdot N(P,x)\n", @@ -4276,9 +4158,7 @@ { "cell_type": "markdown", "id": "1ba4c8b3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The analytical solution for this problem is" ] @@ -4286,9 +4166,7 @@ { "cell_type": "markdown", "id": "e8faf0e1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "g(x) = x(1 - x)\\exp(x)\n", @@ -4298,9 +4176,7 @@ { "cell_type": "markdown", "id": "e4ce498c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Solving the equation using Autograd" ] @@ -4309,10 +4185,7 @@ "cell_type": "code", "execution_count": 28, "id": "1d1cb692", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -4470,9 +4343,7 @@ { "cell_type": "markdown", "id": "16aa2193", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Comparing with a numerical scheme\n", "\n", @@ -4492,9 +4363,7 @@ { "cell_type": "markdown", "id": "b3ef1c2b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -4509,9 +4378,7 @@ { "cell_type": "markdown", "id": "0e1c476f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "If $x_i = i \\Delta x = x_{i-1} + \\Delta x$ and $g_i = g(x_i)$ for $i = 1,\\dots N_x - 2$ with $N_x$ being the number of values for $x$, ([15](#approx)) becomes" ] @@ -4519,9 +4386,7 @@ { "cell_type": "markdown", "id": "7b2ad33d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{aligned}\n", @@ -4534,9 +4399,7 @@ { "cell_type": "markdown", "id": "1c57f992", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Since we know from our problem that" ] @@ -4544,9 +4407,7 @@ { "cell_type": "markdown", "id": "99e9dc9f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{aligned}\n", @@ -4559,9 +4420,7 @@ { "cell_type": "markdown", "id": "ecc62734", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "along with the conditions $g(0) = g(1) = 0$,\n", "the following scheme can be used to find an approximate solution for $g(x)$ numerically:" @@ -4570,9 +4429,7 @@ { "cell_type": "markdown", "id": "2d467ec2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -4590,9 +4447,7 @@ { "cell_type": "markdown", "id": "2de58f3a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "for $i = 1, \\dots, N_x - 2$ where $g_0 = g_{N_x - 1} = 0$ and $f(x_i) = (3x_i + x_i^2)\\exp(x_i)$, which is given for our specific problem.\n", "\n", @@ -4602,9 +4457,7 @@ { "cell_type": "markdown", "id": "0182bb77", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{aligned}\n", @@ -4639,9 +4492,7 @@ { "cell_type": "markdown", "id": "7ef6f3b0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which makes it possible to solve for the vector $\\boldsymbol{g}$." ] @@ -4649,9 +4500,7 @@ { "cell_type": "markdown", "id": "6414011b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Setting up the code\n", "\n", @@ -4662,10 +4511,7 @@ "cell_type": "code", "execution_count": 29, "id": "dc3a8d10", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -4863,9 +4709,7 @@ { "cell_type": "markdown", "id": "4b05b79c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Partial Differential Equations\n", "\n", @@ -4880,9 +4724,7 @@ { "cell_type": "markdown", "id": "0f1b8d34", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -4897,9 +4739,7 @@ { "cell_type": "markdown", "id": "358a4063", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $f$ is an expression involving all kinds of possible mixed derivatives of $g(x_1,\\dots,x_N)$ up to an order $n$. In order for the solution to be unique, some additional conditions must also be given." ] @@ -4907,9 +4747,7 @@ { "cell_type": "markdown", "id": "e4009b2b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Type of problem\n", "\n", @@ -4922,9 +4760,7 @@ { "cell_type": "markdown", "id": "c5a9ee8a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{align*}\n", @@ -4936,9 +4772,7 @@ { "cell_type": "markdown", "id": "b300ac7d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $h_1(x_1,\\dots,x_N)$ is a function that ensures $g_t(x_1,\\dots,x_N)$ satisfies some given conditions.\n", "The neural network $N(x_1,\\dots,x_N,P)$ has weights and biases described by $P$ and $h_2(x_1,\\dots,x_N,N(x_1,\\dots,x_N,P))$ is an expression using the output from the neural network in some way.\n", @@ -4949,9 +4783,7 @@ { "cell_type": "markdown", "id": "ee3d4dba", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Network requirements\n", "\n", @@ -4969,9 +4801,7 @@ { "cell_type": "markdown", "id": "73ef05ff", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C\\left(x_1, \\dots, x_N, P\\right) = \\left( f\\left(x_1, \\, \\dots \\, , x_N, \\frac{\\partial g(x_1,\\dots,x_N) }{\\partial x_1}, \\dots , \\frac{\\partial g(x_1,\\dots,x_N) }{\\partial x_N}, \\frac{\\partial g(x_1,\\dots,x_N) }{\\partial x_1\\partial x_2}, \\, \\dots \\, , \\frac{\\partial^n g(x_1,\\dots,x_N) }{\\partial x_N^n} \\right) \\right)^2\n", @@ -4981,9 +4811,7 @@ { "cell_type": "markdown", "id": "bc7a6597", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## More details\n", "\n", @@ -4993,9 +4821,7 @@ { "cell_type": "markdown", "id": "a158dec6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C\\left(\\boldsymbol{x}, P\\right) = f\\left( \\left( \\boldsymbol{x}, \\frac{\\partial g(\\boldsymbol{x}) }{\\partial x_1}, \\dots , \\frac{\\partial g(\\boldsymbol{x}) }{\\partial x_N}, \\frac{\\partial g(\\boldsymbol{x}) }{\\partial x_1\\partial x_2}, \\, \\dots \\, , \\frac{\\partial^n g(\\boldsymbol{x}) }{\\partial x_N^n} \\right) \\right)^2\n", @@ -5005,9 +4831,7 @@ { "cell_type": "markdown", "id": "6cbfa1b1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "If we also have $M$ different sets of values for $x_1, \\dots, x_N$, that is $\\boldsymbol{x}_i = \\big(x_1^{(i)}, \\dots, x_N^{(i)}\\big)$ for $i = 1,\\dots,M$ being the rows in matrix $X$, the cost function can be generalized into" ] @@ -5015,9 +4839,7 @@ { "cell_type": "markdown", "id": "a3af3492", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C\\left(X, P \\right) = \\sum_{i=1}^M f\\left( \\left( \\boldsymbol{x}_i, \\frac{\\partial g(\\boldsymbol{x}_i) }{\\partial x_1}, \\dots , \\frac{\\partial g(\\boldsymbol{x}_i) }{\\partial x_N}, \\frac{\\partial g(\\boldsymbol{x}_i) }{\\partial x_1\\partial x_2}, \\, \\dots \\, , \\frac{\\partial^n g(\\boldsymbol{x}_i) }{\\partial x_N^n} \\right) \\right)^2.\n", @@ -5027,9 +4849,7 @@ { "cell_type": "markdown", "id": "ccd61199", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example: The diffusion equation\n", "\n", @@ -5039,9 +4859,7 @@ { "cell_type": "markdown", "id": "dc36a5cb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial g(x,t)}{\\partial t} = \\frac{\\partial^2 g(x,t)}{\\partial x^2}\n", @@ -5051,9 +4869,7 @@ { "cell_type": "markdown", "id": "a8b26da0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where a possible choice of conditions are" ] @@ -5061,9 +4877,7 @@ { "cell_type": "markdown", "id": "41406843", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{align*}\n", @@ -5077,9 +4891,7 @@ { "cell_type": "markdown", "id": "f45dd83f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $u(x)$ being some given function." ] @@ -5087,9 +4899,7 @@ { "cell_type": "markdown", "id": "e30bdb98", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Defining the problem\n", "\n", @@ -5099,9 +4909,7 @@ { "cell_type": "markdown", "id": "f9cdb4e5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -5116,9 +4924,7 @@ { "cell_type": "markdown", "id": "16dc73b3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and" ] @@ -5126,9 +4932,7 @@ { "cell_type": "markdown", "id": "ee4dbc98", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{align*}\n", @@ -5142,9 +4946,7 @@ { "cell_type": "markdown", "id": "13748484", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $u(x) = \\sin(\\pi x)$.\n", "\n", @@ -5156,9 +4958,7 @@ { "cell_type": "markdown", "id": "4b26b938", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Setting up the network using Autograd\n", "\n", @@ -5175,10 +4975,7 @@ "cell_type": "code", "execution_count": 30, "id": "9e3bba2f", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "def sigmoid(z):\n", @@ -5230,9 +5027,7 @@ { "cell_type": "markdown", "id": "c4fffb7e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Setting up the network using Autograd; The trial solution\n", "\n", @@ -5260,9 +5055,7 @@ { "cell_type": "markdown", "id": "3ee8799d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Why the jacobian?\n", "\n", @@ -5289,10 +5082,7 @@ "cell_type": "code", "execution_count": 31, "id": "8a2377ff", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# Set up the trial function:\n", @@ -5336,9 +5126,7 @@ { "cell_type": "markdown", "id": "8dbd62e1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Setting up the network using Autograd; The full program\n", "\n", @@ -5362,10 +5150,7 @@ "cell_type": "code", "execution_count": 32, "id": "61b77dd3", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -5596,9 +5381,7 @@ { "cell_type": "markdown", "id": "41169c76", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example: Solving the wave equation with Neural Networks\n", "\n", @@ -5608,9 +5391,7 @@ { "cell_type": "markdown", "id": "3bc273de", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial^2 g(x,t)}{\\partial t^2} = c^2\\frac{\\partial^2 g(x,t)}{\\partial x^2}\n", @@ -5620,9 +5401,7 @@ { "cell_type": "markdown", "id": "37896dc3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $c$ being the specified wave speed.\n", "\n", @@ -5632,9 +5411,7 @@ { "cell_type": "markdown", "id": "09477c51", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{align*}\n", @@ -5649,9 +5426,7 @@ { "cell_type": "markdown", "id": "93579057", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\frac{\\partial g(x,t)}{\\partial t} \\Big |_{t = 0}$ means the derivative of $g(x,t)$ with respect to $t$ is evaluated at $t = 0$, and $u(x)$ and $v(x)$ being given functions." ] @@ -5659,9 +5434,7 @@ { "cell_type": "markdown", "id": "dcbb82f6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The problem to solve for\n", "\n", @@ -5671,9 +5444,7 @@ { "cell_type": "markdown", "id": "f91e973a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -5688,9 +5459,7 @@ { "cell_type": "markdown", "id": "e13ab50c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $c$ is the given wave speed.\n", "The chosen conditions for this equation are" @@ -5699,9 +5468,7 @@ { "cell_type": "markdown", "id": "fca02e71", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -5719,9 +5486,7 @@ { "cell_type": "markdown", "id": "4bb32ee3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In this example, let $c = 1$ and $u(x) = \\sin(\\pi x)$ and $v(x) = -\\pi\\sin(\\pi x)$." ] @@ -5729,9 +5494,7 @@ { "cell_type": "markdown", "id": "feea59f2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The trial solution\n", "Setting up the network is done in similar matter as for the example of solving the diffusion equation.\n", @@ -5755,9 +5518,7 @@ { "cell_type": "markdown", "id": "880753aa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The analytical solution\n", "\n", @@ -5771,9 +5532,7 @@ { "cell_type": "markdown", "id": "42a69cb7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Solving the wave equation - the full program using Autograd" ] @@ -5782,10 +5541,7 @@ "cell_type": "code", "execution_count": 33, "id": "1b321e14", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -6013,9 +5769,7 @@ { "cell_type": "markdown", "id": "c81c7c8a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Resources on differential equations and deep learning\n", "\n", @@ -6029,7 +5783,25 @@ ] } ], - "metadata": {}, + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.10" + } + }, "nbformat": 4, "nbformat_minor": 5 } diff --git a/doc/LectureNotes/_build/html/exercisesweek43.html b/doc/LectureNotes/_build/html/exercisesweek43.html index 5fd64186c..75ff3d0ad 100644 --- a/doc/LectureNotes/_build/html/exercisesweek43.html +++ b/doc/LectureNotes/_build/html/exercisesweek43.html @@ -451,6 +451,53 @@ const thebe_selector_output = ".output, .cell_output" Representing the Data Sets +
  • + + Setting up the Neural Network + +
    • +
  • + + The Code using Scikit-Learn + +
    • +
  • + + Building a neural network code + + +
    • @@ -493,6 +540,53 @@ const thebe_selector_output = ".output, .cell_output" Representing the Data Sets +
  • + + Setting up the Neural Network + +
    • +
  • + + The Code using Scikit-Learn + +
    • +
  • + + Building a neural network code + + +
    • @@ -508,7 +602,7 @@ const thebe_selector_output = ".output, .cell_output" doconce format html exercisesweek43.do.txt -->

    Exercises weeks 43 and 44

    -

    October 9-13, 2023

    +

    October 23-27, 2023

    Date: Deadline is Sunday November 5 at midnight

    You can hand in the exercises from week 43 and week 44 as one exercise and get a total score of two additional points.

    @@ -589,6 +683,13781 @@ inputs \(x_1\) and

    Everything you develop here can be used directly into the code for the project.

    +
    +

    Setting up the Neural Network

    +

    We define first our design matrix and the various output vectors for the different gates.

    +
    +
    +
    %matplotlib inline
+
+"""
+Simple code that tests XOR, OR and AND gates with linear regression
+"""
+
+# import necessary packages
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn import datasets
+
+def sigmoid(x):
+    return 1/(1 + np.exp(-x))
+
+def feed_forward(X):
+    # weighted sum of inputs to the hidden layer
+    z_h = np.matmul(X, hidden_weights) + hidden_bias
+    # activation in the hidden layer
+    a_h = sigmoid(z_h)
+    
+    # weighted sum of inputs to the output layer
+    z_o = np.matmul(a_h, output_weights) + output_bias
+    # softmax output
+    # axis 0 holds each input and axis 1 the probabilities of each category
+    probabilities = sigmoid(z_o)
+    return probabilities
+
+# we obtain a prediction by taking the class with the highest likelihood
+def predict(X):
+    probabilities = feed_forward(X)
+    return np.argmax(probabilities, axis=1)
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+# Design matrix
+X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)
+
+# The XOR gate
+yXOR = np.array( [ 0, 1 ,1, 0])
+# The OR gate
+yOR = np.array( [ 0, 1 ,1, 1])
+# The AND gate
+yAND = np.array( [ 0, 0 ,0, 1])
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 2
+n_features = 2
+
+# we make the weights normally distributed using numpy.random.randn
+
+# weights and bias in the hidden layer
+hidden_weights = np.random.randn(n_features, n_hidden_neurons)
+hidden_bias = np.zeros(n_hidden_neurons) + 0.01
+
+# weights and bias in the output layer
+output_weights = np.random.randn(n_hidden_neurons, n_categories)
+output_bias = np.zeros(n_categories) + 0.01
+
+probabilities = feed_forward(X)
+print(probabilities)
+
+
+predictions = predict(X)
+print(predictions)
+
    +
    +
    +
    +
    [[0.80625657 0.36420967]
+ [0.90297441 0.30170017]
+ [0.89823921 0.28566769]
+ [0.93420126 0.25920793]]
+[0 0 0 0]
+
    +
    +
    +
    +

    Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above.

    +
    +
    +

    The Code using Scikit-Learn

    +
    +
    +
    # import necessary packages
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.neural_network import MLPClassifier
+from sklearn.metrics import accuracy_score
+import seaborn as sns
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+# Design matrix
+X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)
+
+# The XOR gate
+yXOR = np.array( [ 0, 1 ,1, 0])
+# The OR gate
+yOR = np.array( [ 0, 1 ,1, 1])
+# The AND gate
+yAND = np.array( [ 0, 0 ,0, 1])
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 2
+n_features = 2
+
+eta_vals = np.logspace(-5, 1, 7)
+lmbd_vals = np.logspace(-5, 1, 7)
+# store models for later use
+DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
+epochs = 100
+
+for i, eta in enumerate(eta_vals):
+    for j, lmbd in enumerate(lmbd_vals):
+        dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',
+                            alpha=lmbd, learning_rate_init=eta, max_iter=epochs)
+        dnn.fit(X, yXOR)
+        DNN_scikit[i][j] = dnn
+        print("Learning rate  = ", eta)
+        print("Lambda = ", lmbd)
+        print("Accuracy score on data set: ", dnn.score(X, yXOR))
+        print()
+
+sns.set()
+test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+for i in range(len(eta_vals)):
+    for j in range(len(lmbd_vals)):
+        dnn = DNN_scikit[i][j]
+        test_pred = dnn.predict(X)
+        test_accuracy[i][j] = accuracy_score(yXOR, test_pred)
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Test Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+
    +
    +
    +
    +
    Learning rate  =  1e-05
+Lambda =  1e-05
+Accuracy score on data set:  0.5
+
+Learning rate  =  1e-05
+Lambda =  0.0001
+Accuracy score on data set:  0.5
+
+Learning rate  =  1e-05
+Lambda =  0.001
+Accuracy score on data set:  0.5
+
+Learning rate  =  1e-05
+Lambda =  0.01
+Accuracy score on data set:  0.5
+
+Learning rate  =  1e-05
+Lambda =  0.1
+Accuracy score on data set:  0.5
+
+Learning rate  =  1e-05
+Lambda =  1.0
+Accuracy score on data set:  0.5
+
+Learning rate  =  1e-05
+Lambda =  10.0
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.0001
+Lambda =  1e-05
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.0001
+Lambda =  0.0001
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.0001
+Lambda =  0.001
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.0001
+Lambda =  0.01
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.0001
+Lambda =  0.1
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.0001
+Lambda =  1.0
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.0001
+Lambda =  10.0
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.001
+Lambda =  1e-05
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.001
+Lambda =  0.0001
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.001
+Lambda =  0.001
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.001
+Lambda =  0.01
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.001
+Lambda =  0.1
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.001
+Lambda =  1.0
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.001
+Lambda =  10.0
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.01
+Lambda =  1e-05
+Accuracy score on data set:  0.25
+
+Learning rate  =  0.01
+Lambda =  0.0001
+Accuracy score on data set:  0.75
+
+Learning rate  =  0.01
+Lambda =  0.001
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.01
+Lambda =  0.01
+Accuracy score on data set:  0.75
+
+Learning rate  =  0.01
+Lambda =  0.1
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.01
+Lambda =  1.0
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.01
+Lambda =  10.0
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.1
+Lambda =  1e-05
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.1
+Lambda =  0.0001
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.1
+Lambda =  0.001
+Accuracy score on data set:  1.0
+
+Learning rate  =  0.1
+Lambda =  0.01
+Accuracy score on data set:  1.0
+
+Learning rate  =  0.1
+Lambda =  0.1
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.1
+Lambda =  1.0
+Accuracy score on data set:  0.5
+
+Learning rate  =  0.1
+Lambda =  10.0
+Accuracy score on data set:  0.5
+
+Learning rate  =  1.0
+Lambda =  1e-05
+Accuracy score on data set:  0.75
+
+Learning rate  =  1.0
+Lambda =  0.0001
+Accuracy score on data set:  0.75
+
+Learning rate  =  1.0
+Lambda =  0.001
+Accuracy score on data set:  0.75
+
+Learning rate  =  1.0
+Lambda =  0.01
+Accuracy score on data set:  0.5
+
+Learning rate  =  1.0
+Lambda =  0.1
+Accuracy score on data set:  0.5
+
+Learning rate  =  1.0
+Lambda =  1.0
+Accuracy score on data set:  0.5
+
+Learning rate  =  1.0
+Lambda =  10.0
+Accuracy score on data set:  0.5
+
+Learning rate  =  10.0
+Lambda =  1e-05
+Accuracy score on data set:  0.5
+
+Learning rate  =  10.0
+Lambda =  0.0001
+Accuracy score on data set:  0.5
+
+Learning rate  =  10.0
+Lambda =  0.001
+Accuracy score on data set:  0.5
+
+Learning rate  =  10.0
+Lambda =  0.01
+Accuracy score on data set:  0.5
+
+Learning rate  =  10.0
+Lambda =  0.1
+Accuracy score on data set:  0.5
+
+Learning rate  =  10.0
+Lambda =  1.0
+Accuracy score on data set:  0.5
+
+Learning rate  =  10.0
+Lambda =  10.0
+Accuracy score on data set:  0.5
+
    +
    +
    /Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
+  warnings.warn(
+/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
+  warnings.warn(
+/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
+  warnings.warn(
+/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
+  warnings.warn(
+/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
+  warnings.warn(
+/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
+  warnings.warn(
+/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
+  warnings.warn(
+/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
+  warnings.warn(
+/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
+  warnings.warn(
+/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
+  warnings.warn(
+
    +
    +_images/exercisesweek43_11_2.png +
    +
    +
    +
    +

    Building a neural network code

    +

    Here we present a flexible object oriented codebase +for a feed forward neural network, along with a demonstration of how +to use it. Before we get into the details of the neural network, we +will first present some implementations of various schedulers, cost +functions and activation functions that can be used together with the +neural network.

    +
    +

    Learning rate methods

    +

    The code below shows object oriented implementations of the Constant, +Momentum, Adagrad, AdagradMomentum, RMS prop and Adam schedulers. All +of the classes belong to the shared abstract Scheduler class, and +share the update_change() and reset() methods allowing for any of the +schedulers to be seamlessly used during the training stage, as will +later be shown in the fit() method of the neural +network. Update_change() only has one parameter, the gradient +(\(δ^l_ja^{l−1}_k\)), and returns the change which will be subtracted +from the weights. The reset() function takes no parameters, and resets +the desired variables. For Constant and Momentum, reset does nothing.

    +
    +
    +
    import autograd.numpy as np
+
+class Scheduler:
+    """
+    Abstract class for Schedulers
+    """
+
+    def __init__(self, eta):
+        self.eta = eta
+
+    # should be overwritten
+    def update_change(self, gradient):
+        raise NotImplementedError
+
+    # overwritten if needed
+    def reset(self):
+        pass
+
+
+class Constant(Scheduler):
+    def __init__(self, eta):
+        super().__init__(eta)
+
+    def update_change(self, gradient):
+        return self.eta * gradient
+    
+    def reset(self):
+        pass
+
+
+class Momentum(Scheduler):
+    def __init__(self, eta: float, momentum: float):
+        super().__init__(eta)
+        self.momentum = momentum
+        self.change = 0
+
+    def update_change(self, gradient):
+        self.change = self.momentum * self.change + self.eta * gradient
+        return self.change
+
+    def reset(self):
+        pass
+
+
+class Adagrad(Scheduler):
+    def __init__(self, eta):
+        super().__init__(eta)
+        self.G_t = None
+
+    def update_change(self, gradient):
+        delta = 1e-8  # avoid division ny zero
+
+        if self.G_t is None:
+            self.G_t = np.zeros((gradient.shape[0], gradient.shape[0]))
+
+        self.G_t += gradient @ gradient.T
+
+        G_t_inverse = 1 / (
+            delta + np.sqrt(np.reshape(np.diagonal(self.G_t), (self.G_t.shape[0], 1)))
+        )
+        return self.eta * gradient * G_t_inverse
+
+    def reset(self):
+        self.G_t = None
+
+
+class AdagradMomentum(Scheduler):
+    def __init__(self, eta, momentum):
+        super().__init__(eta)
+        self.G_t = None
+        self.momentum = momentum
+        self.change = 0
+
+    def update_change(self, gradient):
+        delta = 1e-8  # avoid division ny zero
+
+        if self.G_t is None:
+            self.G_t = np.zeros((gradient.shape[0], gradient.shape[0]))
+
+        self.G_t += gradient @ gradient.T
+
+        G_t_inverse = 1 / (
+            delta + np.sqrt(np.reshape(np.diagonal(self.G_t), (self.G_t.shape[0], 1)))
+        )
+        self.change = self.change * self.momentum + self.eta * gradient * G_t_inverse
+        return self.change
+
+    def reset(self):
+        self.G_t = None
+
+
+class RMS_prop(Scheduler):
+    def __init__(self, eta, rho):
+        super().__init__(eta)
+        self.rho = rho
+        self.second = 0.0
+
+    def update_change(self, gradient):
+        delta = 1e-8  # avoid division ny zero
+        self.second = self.rho * self.second + (1 - self.rho) * gradient * gradient
+        return self.eta * gradient / (np.sqrt(self.second + delta))
+
+    def reset(self):
+        self.second = 0.0
+
+
+class Adam(Scheduler):
+    def __init__(self, eta, rho, rho2):
+        super().__init__(eta)
+        self.rho = rho
+        self.rho2 = rho2
+        self.moment = 0
+        self.second = 0
+        self.n_epochs = 1
+
+    def update_change(self, gradient):
+        delta = 1e-8  # avoid division ny zero
+
+        self.moment = self.rho * self.moment + (1 - self.rho) * gradient
+        self.second = self.rho2 * self.second + (1 - self.rho2) * gradient * gradient
+
+        moment_corrected = self.moment / (1 - self.rho**self.n_epochs)
+        second_corrected = self.second / (1 - self.rho2**self.n_epochs)
+
+        return self.eta * moment_corrected / (np.sqrt(second_corrected + delta))
+
+    def reset(self):
+        self.n_epochs += 1
+        self.moment = 0
+        self.second = 0
+
    +
    +
    +
    +
    +
    +

    Usage of the above learning rate schedulers

    +

    To initalize a scheduler, simply create the object and pass in the +necessary parameters such as the learning rate and the momentum as +shown below. As the Scheduler class is an abstract class it should not +called directly, and will raise an error upon usage.

    +
    +
    +
    momentum_scheduler = Momentum(eta=1e-3, momentum=0.9)
+adam_scheduler = Adam(eta=1e-3, rho=0.9, rho2=0.999)
+
    +
    +
    +
    +

    Here is a small example for how a segment of code using schedulers +could look. Switching out the schedulers is simple.

    +
    +
    +
    weights = np.ones((3,3))
+print(f"Before scheduler:\n{weights=}")
+
+epochs = 10
+for e in range(epochs):
+    gradient = np.random.rand(3, 3)
+    change = adam_scheduler.update_change(gradient)
+    weights = weights - change
+    adam_scheduler.reset()
+
+print(f"\nAfter scheduler:\n{weights=}")
+
    +
    +
    +
    +
    Before scheduler:
+weights=array([[1., 1., 1.],
+       [1., 1., 1.],
+       [1., 1., 1.]])
+
+After scheduler:
+weights=array([[0.993993  , 0.993993  , 0.99399301],
+       [0.99399308, 0.99399315, 0.99399301],
+       [0.99399301, 0.99399309, 0.99399301]])
+
    +
    +
    +
    +
    +
    +

    Cost functions

    +

    Here we discuss cost functions that can be used when creating the +neural network. Every cost function takes the target vector as its +parameter, and returns a function valued only at \(x\) such that it may +easily be differentiated.

    +
    +
    +
    import autograd.numpy as np
+
+def CostOLS(target):
+    
+    def func(X):
+        return (1.0 / target.shape[0]) * np.sum((target - X) ** 2)
+
+    return func
+
+
+def CostLogReg(target):
+
+    def func(X):
+        
+        return -(1.0 / target.shape[0]) * np.sum(
+            (target * np.log(X + 10e-10)) + ((1 - target) * np.log(1 - X + 10e-10))
+        )
+
+    return func
+
+
+def CostCrossEntropy(target):
+    
+    def func(X):
+        return -(1.0 / target.size) * np.sum(target * np.log(X + 10e-10))
+
+    return func
+
    +
    +
    +
    +

    Below we give a short example of how these cost function may be used +to obtain results if you wish to test them out on your own using +AutoGrad’s automatics differentiation.

    +
    +
    +
    from autograd import grad
+
+target = np.array([[1, 2, 3]]).T
+a = np.array([[4, 5, 6]]).T
+
+cost_func = CostCrossEntropy
+cost_func_derivative = grad(cost_func(target))
+
+valued_at_a = cost_func_derivative(a)
+print(f"Derivative of cost function {cost_func.__name__} valued at a:\n{valued_at_a}")
+
    +
    +
    +
    +
    Derivative of cost function CostCrossEntropy valued at a:
+[[-0.08333333]
+ [-0.13333333]
+ [-0.16666667]]
+
    +
    +
    +
    +
    +
    +

    Activation functions

    +

    Finally, before we look at the neural network, we will look at the +activation functions which can be specified between the hidden layers +and as the output function. Each function can be valued for any given +vector or matrix X, and can be differentiated via derivate().

    +
    +
    +
    import autograd.numpy as np
+from autograd import elementwise_grad
+
+def identity(X):
+    return X
+
+
+def sigmoid(X):
+    try:
+        return 1.0 / (1 + np.exp(-X))
+    except FloatingPointError:
+        return np.where(X > np.zeros(X.shape), np.ones(X.shape), np.zeros(X.shape))
+
+
+def softmax(X):
+    X = X - np.max(X, axis=-1, keepdims=True)
+    delta = 10e-10
+    return np.exp(X) / (np.sum(np.exp(X), axis=-1, keepdims=True) + delta)
+
+
+def RELU(X):
+    return np.where(X > np.zeros(X.shape), X, np.zeros(X.shape))
+
+
+def LRELU(X):
+    delta = 10e-4
+    return np.where(X > np.zeros(X.shape), X, delta * X)
+
+
+def derivate(func):
+    if func.__name__ == "RELU":
+
+        def func(X):
+            return np.where(X > 0, 1, 0)
+
+        return func
+
+    elif func.__name__ == "LRELU":
+
+        def func(X):
+            delta = 10e-4
+            return np.where(X > 0, 1, delta)
+
+        return func
+
+    else:
+        return elementwise_grad(func)
+
    +
    +
    +
    +

    Below follows a short demonstration of how to use an activation +function. The derivative of the activation function will be important +when calculating the output delta term during backpropagation. Note +that derivate() can also be used for cost functions for a more +generalized approach.

    +
    +
    +
    z = np.array([[4, 5, 6]]).T
+print(f"Input to activation function:\n{z}")
+
+act_func = sigmoid
+a = act_func(z)
+print(f"\nOutput from {act_func.__name__} activation function:\n{a}")
+
+act_func_derivative = derivate(act_func)
+valued_at_z = act_func_derivative(a)
+print(f"\nDerivative of {act_func.__name__} activation function valued at z:\n{valued_at_z}")
+
    +
    +
    +
    +
    Input to activation function:
+[[4]
+ [5]
+ [6]]
+
+Output from sigmoid activation function:
+[[0.98201379]
+ [0.99330715]
+ [0.99752738]]
+
+Derivative of sigmoid activation function valued at z:
+[[0.19824029]
+ [0.19721923]
+ [0.19683648]]
+
    +
    +
    +
    +
    +
    +

    The Neural Network

    +

    Now that we have gotten a good understanding of the implementation of +some important components, we can take a look at an object oriented +implementation of a feed forward neural network. The feed forward +neural network has been implemented as a class named FFNN, which can +be initiated as a regressor or classifier dependant on the choice of +cost function. The FFNN can have any number of input nodes, hidden +layers with any amount of hidden nodes, and any amount of output nodes +meaning it can perform multiclass classification as well as binary +classification and regression problems. Although there is a lot of +code present, it makes for an easy to use and generalizeable interface +for creating many types of neural networks as will be demonstrated +below.

    +
    +
    +
    import math
+import autograd.numpy as np
+import sys
+import warnings
+from autograd import grad, elementwise_grad
+from random import random, seed
+from copy import deepcopy, copy
+from typing import Tuple, Callable
+from sklearn.utils import resample
+
+warnings.simplefilter("error")
+
+
+class FFNN:
+    """
+    Description:
+    ------------
+        Feed Forward Neural Network with interface enabling flexible design of a
+        nerual networks architecture and the specification of activation function
+        in the hidden layers and output layer respectively. This model can be used
+        for both regression and classification problems, depending on the output function.
+
+    Attributes:
+    ------------
+        I   dimensions (tuple[int]): A list of positive integers, which specifies the
+            number of nodes in each of the networks layers. The first integer in the array
+            defines the number of nodes in the input layer, the second integer defines number
+            of nodes in the first hidden layer and so on until the last number, which
+            specifies the number of nodes in the output layer.
+        II  hidden_func (Callable): The activation function for the hidden layers
+        III output_func (Callable): The activation function for the output layer
+        IV  cost_func (Callable): Our cost function
+        V   seed (int): Sets random seed, makes results reproducible
+    """
+
+    def __init__(
+        self,
+        dimensions: tuple[int],
+        hidden_func: Callable = sigmoid,
+        output_func: Callable = lambda x: x,
+        cost_func: Callable = CostOLS,
+        seed: int = None,
+    ):
+        self.dimensions = dimensions
+        self.hidden_func = hidden_func
+        self.output_func = output_func
+        self.cost_func = cost_func
+        self.seed = seed
+        self.weights = list()
+        self.schedulers_weight = list()
+        self.schedulers_bias = list()
+        self.a_matrices = list()
+        self.z_matrices = list()
+        self.classification = None
+
+        self.reset_weights()
+        self._set_classification()
+
+    def fit(
+        self,
+        X: np.ndarray,
+        t: np.ndarray,
+        scheduler: Scheduler,
+        batches: int = 1,
+        epochs: int = 100,
+        lam: float = 0,
+        X_val: np.ndarray = None,
+        t_val: np.ndarray = None,
+    ):
+        """
+        Description:
+        ------------
+            This function performs the training the neural network by performing the feedforward and backpropagation
+            algorithm to update the networks weights.
+
+        Parameters:
+        ------------
+            I    X (np.ndarray) : training data
+            II   t (np.ndarray) : target data
+            III  scheduler (Scheduler) : specified scheduler (algorithm for optimization of gradient descent)
+            IV   scheduler_args (list[int]) : list of all arguments necessary for scheduler
+
+        Optional Parameters:
+        ------------
+            V    batches (int) : number of batches the datasets are split into, default equal to 1
+            VI   epochs (int) : number of iterations used to train the network, default equal to 100
+            VII  lam (float) : regularization hyperparameter lambda
+            VIII X_val (np.ndarray) : validation set
+            IX   t_val (np.ndarray) : validation target set
+
+        Returns:
+        ------------
+            I   scores (dict) : A dictionary containing the performance metrics of the model.
+                The number of the metrics depends on the parameters passed to the fit-function.
+
+        """
+
+        # setup 
+        if self.seed is not None:
+            np.random.seed(self.seed)
+
+        val_set = False
+        if X_val is not None and t_val is not None:
+            val_set = True
+
+        # creating arrays for score metrics
+        train_errors = np.empty(epochs)
+        train_errors.fill(np.nan)
+        val_errors = np.empty(epochs)
+        val_errors.fill(np.nan)
+
+        train_accs = np.empty(epochs)
+        train_accs.fill(np.nan)
+        val_accs = np.empty(epochs)
+        val_accs.fill(np.nan)
+
+        self.schedulers_weight = list()
+        self.schedulers_bias = list()
+
+        batch_size = X.shape[0] // batches
+
+        X, t = resample(X, t)
+
+        # this function returns a function valued only at X
+        cost_function_train = self.cost_func(t)
+        if val_set:
+            cost_function_val = self.cost_func(t_val)
+
+        # create schedulers for each weight matrix
+        for i in range(len(self.weights)):
+            self.schedulers_weight.append(copy(scheduler))
+            self.schedulers_bias.append(copy(scheduler))
+
+        print(f"{scheduler.__class__.__name__}: Eta={scheduler.eta}, Lambda={lam}")
+
+        try:
+            for e in range(epochs):
+                for i in range(batches):
+                    # allows for minibatch gradient descent
+                    if i == batches - 1:
+                        # If the for loop has reached the last batch, take all thats left
+                        X_batch = X[i * batch_size :, :]
+                        t_batch = t[i * batch_size :, :]
+                    else:
+                        X_batch = X[i * batch_size : (i + 1) * batch_size, :]
+                        t_batch = t[i * batch_size : (i + 1) * batch_size, :]
+
+                    self._feedforward(X_batch)
+                    self._backpropagate(X_batch, t_batch, lam)
+
+                # reset schedulers for each epoch (some schedulers pass in this call)
+                for scheduler in self.schedulers_weight:
+                    scheduler.reset()
+
+                for scheduler in self.schedulers_bias:
+                    scheduler.reset()
+
+                # computing performance metrics
+                pred_train = self.predict(X)
+                train_error = cost_function_train(pred_train)
+
+                train_errors[e] = train_error
+                if val_set:
+                    
+                    pred_val = self.predict(X_val)
+                    val_error = cost_function_val(pred_val)
+                    val_errors[e] = val_error
+
+                if self.classification:
+                    train_acc = self._accuracy(self.predict(X), t)
+                    train_accs[e] = train_acc
+                    if val_set:
+                        val_acc = self._accuracy(pred_val, t_val)
+                        val_accs[e] = val_acc
+
+                # printing progress bar
+                progression = e / epochs
+                print_length = self._progress_bar(
+                    progression,
+                    train_error=train_errors[e],
+                    train_acc=train_accs[e],
+                    val_error=val_errors[e],
+                    val_acc=val_accs[e],
+                )
+        except KeyboardInterrupt:
+            # allows for stopping training at any point and seeing the result
+            pass
+
+        # visualization of training progression (similiar to tensorflow progression bar)
+        sys.stdout.write("\r" + " " * print_length)
+        sys.stdout.flush()
+        self._progress_bar(
+            1,
+            train_error=train_errors[e],
+            train_acc=train_accs[e],
+            val_error=val_errors[e],
+            val_acc=val_accs[e],
+        )
+        sys.stdout.write("")
+
+        # return performance metrics for the entire run
+        scores = dict()
+
+        scores["train_errors"] = train_errors
+
+        if val_set:
+            scores["val_errors"] = val_errors
+
+        if self.classification:
+            scores["train_accs"] = train_accs
+
+            if val_set:
+                scores["val_accs"] = val_accs
+
+        return scores
+
+    def predict(self, X: np.ndarray, *, threshold=0.5):
+        """
+         Description:
+         ------------
+             Performs prediction after training of the network has been finished.
+
+         Parameters:
+        ------------
+             I   X (np.ndarray): The design matrix, with n rows of p features each
+
+         Optional Parameters:
+         ------------
+             II  threshold (float) : sets minimal value for a prediction to be predicted as the positive class
+                 in classification problems
+
+         Returns:
+         ------------
+             I   z (np.ndarray): A prediction vector (row) for each row in our design matrix
+                 This vector is thresholded if regression=False, meaning that classification results
+                 in a vector of 1s and 0s, while regressions in an array of decimal numbers
+
+        """
+
+        predict = self._feedforward(X)
+
+        if self.classification:
+            return np.where(predict > threshold, 1, 0)
+        else:
+            return predict
+
+    def reset_weights(self):
+        """
+        Description:
+        ------------
+            Resets/Reinitializes the weights in order to train the network for a new problem.
+
+        """
+        if self.seed is not None:
+            np.random.seed(self.seed)
+
+        self.weights = list()
+        for i in range(len(self.dimensions) - 1):
+            weight_array = np.random.randn(
+                self.dimensions[i] + 1, self.dimensions[i + 1]
+            )
+            weight_array[0, :] = np.random.randn(self.dimensions[i + 1]) * 0.01
+
+            self.weights.append(weight_array)
+
+    def _feedforward(self, X: np.ndarray):
+        """
+        Description:
+        ------------
+            Calculates the activation of each layer starting at the input and ending at the output.
+            Each following activation is calculated from a weighted sum of each of the preceeding
+            activations (except in the case of the input layer).
+
+        Parameters:
+        ------------
+            I   X (np.ndarray): The design matrix, with n rows of p features each
+
+        Returns:
+        ------------
+            I   z (np.ndarray): A prediction vector (row) for each row in our design matrix
+        """
+
+        # reset matrices
+        self.a_matrices = list()
+        self.z_matrices = list()
+
+        # if X is just a vector, make it into a matrix
+        if len(X.shape) == 1:
+            X = X.reshape((1, X.shape[0]))
+
+        # Add a coloumn of zeros as the first coloumn of the design matrix, in order
+        # to add bias to our data
+        bias = np.ones((X.shape[0], 1)) * 0.01
+        X = np.hstack([bias, X])
+
+        # a^0, the nodes in the input layer (one a^0 for each row in X - where the
+        # exponent indicates layer number).
+        a = X
+        self.a_matrices.append(a)
+        self.z_matrices.append(a)
+
+        # The feed forward algorithm
+        for i in range(len(self.weights)):
+            if i < len(self.weights) - 1:
+                z = a @ self.weights[i]
+                self.z_matrices.append(z)
+                a = self.hidden_func(z)
+                # bias column again added to the data here
+                bias = np.ones((a.shape[0], 1)) * 0.01
+                a = np.hstack([bias, a])
+                self.a_matrices.append(a)
+            else:
+                try:
+                    # a^L, the nodes in our output layers
+                    z = a @ self.weights[i]
+                    a = self.output_func(z)
+                    self.a_matrices.append(a)
+                    self.z_matrices.append(z)
+                except Exception as OverflowError:
+                    print(
+                        "OverflowError in fit() in FFNN\nHOW TO DEBUG ERROR: Consider lowering your learning rate or scheduler specific parameters such as momentum, or check if your input values need scaling"
+                    )
+
+        # this will be a^L
+        return a
+
+    def _backpropagate(self, X, t, lam):
+        """
+        Description:
+        ------------
+            Performs the backpropagation algorithm. In other words, this method
+            calculates the gradient of all the layers starting at the
+            output layer, and moving from right to left accumulates the gradient until
+            the input layer is reached. Each layers respective weights are updated while
+            the algorithm propagates backwards from the output layer (auto-differentation in reverse mode).
+
+        Parameters:
+        ------------
+            I   X (np.ndarray): The design matrix, with n rows of p features each.
+            II  t (np.ndarray): The target vector, with n rows of p targets.
+            III lam (float32): regularization parameter used to punish the weights in case of overfitting
+
+        Returns:
+        ------------
+            No return value.
+
+        """
+        out_derivative = derivate(self.output_func)
+        hidden_derivative = derivate(self.hidden_func)
+
+        for i in range(len(self.weights) - 1, -1, -1):
+            # delta terms for output
+            if i == len(self.weights) - 1:
+                # for multi-class classification
+                if (
+                    self.output_func.__name__ == "softmax"
+                ):
+                    delta_matrix = self.a_matrices[i + 1] - t
+                # for single class classification
+                else:
+                    cost_func_derivative = grad(self.cost_func(t))
+                    delta_matrix = out_derivative(
+                        self.z_matrices[i + 1]
+                    ) * cost_func_derivative(self.a_matrices[i + 1])
+
+            # delta terms for hidden layer
+            else:
+                delta_matrix = (
+                    self.weights[i + 1][1:, :] @ delta_matrix.T
+                ).T * hidden_derivative(self.z_matrices[i + 1])
+
+            # calculate gradient
+            gradient_weights = self.a_matrices[i][:, 1:].T @ delta_matrix
+            gradient_bias = np.sum(delta_matrix, axis=0).reshape(
+                1, delta_matrix.shape[1]
+            )
+
+            # regularization term
+            gradient_weights += self.weights[i][1:, :] * lam
+
+            # use scheduler
+            update_matrix = np.vstack(
+                [
+                    self.schedulers_bias[i].update_change(gradient_bias),
+                    self.schedulers_weight[i].update_change(gradient_weights),
+                ]
+            )
+
+            # update weights and bias
+            self.weights[i] -= update_matrix
+
+    def _accuracy(self, prediction: np.ndarray, target: np.ndarray):
+        """
+        Description:
+        ------------
+            Calculates accuracy of given prediction to target
+
+        Parameters:
+        ------------
+            I   prediction (np.ndarray): vector of predicitons output network
+                (1s and 0s in case of classification, and real numbers in case of regression)
+            II  target (np.ndarray): vector of true values (What the network ideally should predict)
+
+        Returns:
+        ------------
+            A floating point number representing the percentage of correctly classified instances.
+        """
+        assert prediction.size == target.size
+        return np.average((target == prediction))
+    def _set_classification(self):
+        """
+        Description:
+        ------------
+            Decides if FFNN acts as classifier (True) og regressor (False),
+            sets self.classification during init()
+        """
+        self.classification = False
+        if (
+            self.cost_func.__name__ == "CostLogReg"
+            or self.cost_func.__name__ == "CostCrossEntropy"
+        ):
+            self.classification = True
+
+    def _progress_bar(self, progression, **kwargs):
+        """
+        Description:
+        ------------
+            Displays progress of training
+        """
+        print_length = 40
+        num_equals = int(progression * print_length)
+        num_not = print_length - num_equals
+        arrow = ">" if num_equals > 0 else ""
+        bar = "[" + "=" * (num_equals - 1) + arrow + "-" * num_not + "]"
+        perc_print = self._format(progression * 100, decimals=5)
+        line = f"  {bar} {perc_print}% "
+
+        for key in kwargs:
+            if not np.isnan(kwargs[key]):
+                value = self._format(kwargs[key], decimals=4)
+                line += f"| {key}: {value} "
+        sys.stdout.write("\r" + line)
+        sys.stdout.flush()
+        return len(line)
+
+    def _format(self, value, decimals=4):
+        """
+        Description:
+        ------------
+            Formats decimal numbers for progress bar
+        """
+        if value > 0:
+            v = value
+        elif value < 0:
+            v = -10 * value
+        else:
+            v = 1
+        n = 1 + math.floor(math.log10(v))
+        if n >= decimals - 1:
+            return str(round(value))
+        return f"{value:.{decimals-n-1}f}"
+
    +
    +
    +
    +

    Before we make a model, we will quickly generate a dataset we can use +for our linear regression problem as shown below

    +
    +
    +
    import autograd.numpy as np
+from sklearn.model_selection import train_test_split
+
+def SkrankeFunction(x, y):
+    return np.ravel(0 + 1*x + 2*y + 3*x**2 + 4*x*y + 5*y**2)
+
+def create_X(x, y, n):
+    if len(x.shape) > 1:
+        x = np.ravel(x)
+        y = np.ravel(y)
+
+    N = len(x)
+    l = int((n + 1) * (n + 2) / 2)  # Number of elements in beta
+    X = np.ones((N, l))
+
+    for i in range(1, n + 1):
+        q = int((i) * (i + 1) / 2)
+        for k in range(i + 1):
+            X[:, q + k] = (x ** (i - k)) * (y**k)
+
+    return X
+
+step=0.5
+x = np.arange(0, 1, step)
+y = np.arange(0, 1, step)
+x, y = np.meshgrid(x, y)
+target = SkrankeFunction(x, y)
+target = target.reshape(target.shape[0], 1)
+
+poly_degree=3
+X = create_X(x, y, poly_degree)
+
+X_train, X_test, t_train, t_test = train_test_split(X, target)
+
    +
    +
    +
    +

    Now that we have our dataset ready for the regression, we can create +our regressor. Note that with the seed parameter, we can make sure our +results stay the same every time we run the neural network. For +inititialization, we simply specify the dimensions (we wish the amount +of input nodes to be equal to the datapoints, and the output to +predict one value).

    +
    +
    +
    input_nodes = X_train.shape[1]
+output_nodes = 1
+
+linear_regression = FFNN((input_nodes, output_nodes), output_func=identity, cost_func=CostOLS, seed=2023)
+
    +
    +
    +
    +

    We then fit our model with our training data using the scheduler of our choice.

    +
    +
    +
    linear_regression.reset_weights() # reset weights such that previous runs or reruns don't affect the weights
+
+scheduler = Constant(eta=1e-3)
+scores = linear_regression.fit(X_train, t_train, scheduler)
+
    +
    +
    +
    +
    Constant: Eta=0.001, Lambda=0
+
+  [----------------------------------------] 0.000% | train_error: 3.69 
+
    +
    +
      [----------------------------------------] 1.000% | train_error: 3.67 
+
    +
    +
      [----------------------------------------] 2.000% | train_error: 3.65 
+
    +
    +
      [>---------------------------------------] 3.000% | train_error: 3.64 
+
    +
    +
      [>---------------------------------------] 4.000% | train_error: 3.62 
+
    +
    +
      [=>--------------------------------------] 5.000% | train_error: 3.60 
+
    +
    +
      [=>--------------------------------------] 6.000% | train_error: 3.58 
+
    +
    +
      [=>--------------------------------------] 7.000% | train_error: 3.57 
+
    +
    +
      [==>-------------------------------------] 8.000% | train_error: 3.55 
+
    +
    +
      [==>-------------------------------------] 9.000% | train_error: 3.53 
+
    +
    +
      [===>------------------------------------] 10.00% | train_error: 3.52 
+
    +
    +
      [===>------------------------------------] 11.00% | train_error: 3.50 
+
    +
    +
      [===>------------------------------------] 12.00% | train_error: 3.48 
+
    +
    +
      [====>-----------------------------------] 13.00% | train_error: 3.47 
+
    +
    +
      [====>-----------------------------------] 14.00% | train_error: 3.45 
+
    +
    +
      [=====>----------------------------------] 15.00% | train_error: 3.43 
+
    +
    +
      [=====>----------------------------------] 16.00% | train_error: 3.42 
+
    +
    +
      [=====>----------------------------------] 17.00% | train_error: 3.40 
+
    +
    +
      [======>---------------------------------] 18.00% | train_error: 3.38 
+
    +
    +
      [======>---------------------------------] 19.00% | train_error: 3.37 
+
    +
    +
      [=======>--------------------------------] 20.00% | train_error: 3.35 
+
    +
    +
      [=======>--------------------------------] 21.00% | train_error: 3.34 
+
    +
    +
      [=======>--------------------------------] 22.00% | train_error: 3.32 
+
    +
    +
      [========>-------------------------------] 23.00% | train_error: 3.31 
+
    +
    +
      [========>-------------------------------] 24.00% | train_error: 3.29 
+
    +
    +
      [=========>------------------------------] 25.00% | train_error: 3.27 
+
    +
    +
      [=========>------------------------------] 26.00% | train_error: 3.26 
+
    +
    +
      [=========>------------------------------] 27.00% | train_error: 3.24 
+
    +
    +
      [==========>-----------------------------] 28.00% | train_error: 3.23 
+
    +
    +
      [==========>-----------------------------] 29.00% | train_error: 3.21 
+
    +
    +
      [===========>----------------------------] 30.00% | train_error: 3.20 
+
    +
    +
      [===========>----------------------------] 31.00% | train_error: 3.18 
+
    +
    +
      [===========>----------------------------] 32.00% | train_error: 3.17 
+
    +
    +
      [============>---------------------------] 33.00% | train_error: 3.15 
+
    +
    +
      [============>---------------------------] 34.00% | train_error: 3.14 
+
    +
    +
      [=============>--------------------------] 35.00% | train_error: 3.12 
+
    +
    +
      [=============>--------------------------] 36.00% | train_error: 3.11 
+
    +
    +
      [=============>--------------------------] 37.00% | train_error: 3.09 
+
    +
    +
      [==============>-------------------------] 38.00% | train_error: 3.08 
+
    +
    +
      [==============>-------------------------] 39.00% | train_error: 3.06 
+
    +
    +
      [===============>------------------------] 40.00% | train_error: 3.05 
+
    +
    +
      [===============>------------------------] 41.00% | train_error: 3.03 
+
    +
    +
      [===============>------------------------] 42.00% | train_error: 3.02 
+
    +
    +
      [================>-----------------------] 43.00% | train_error: 3.00 
+
    +
    +
      [================>-----------------------] 44.00% | train_error: 2.99 
+
    +
    +
      [=================>----------------------] 45.00% | train_error: 2.98 
+
    +
    +
      [=================>----------------------] 46.00% | train_error: 2.96 
+
    +
    +
      [=================>----------------------] 47.00% | train_error: 2.95 
+
    +
    +
      [==================>---------------------] 48.00% | train_error: 2.93 
+
    +
    +
      [==================>---------------------] 49.00% | train_error: 2.92 
+
    +
    +
      [===================>--------------------] 50.00% | train_error: 2.91 
+
    +
    +
      [===================>--------------------] 51.00% | train_error: 2.89 
+
    +
    +
      [===================>--------------------] 52.00% | train_error: 2.88 
+
    +
    +
      [====================>-------------------] 53.00% | train_error: 2.86 
+
    +
    +
      [====================>-------------------] 54.00% | train_error: 2.85 
+
    +
    +
      [=====================>------------------] 55.00% | train_error: 2.84 
+
    +
    +
      [=====================>------------------] 56.00% | train_error: 2.82 
+
    +
    +
      [=====================>------------------] 57.00% | train_error: 2.81 
+
    +
    +
      [======================>-----------------] 58.00% | train_error: 2.80 
+
    +
    +
      [======================>-----------------] 59.00% | train_error: 2.78 
+
    +
    +
      [=======================>----------------] 60.00% | train_error: 2.77 
+
    +
    +
      [=======================>----------------] 61.00% | train_error: 2.76 
+
    +
    +
      [=======================>----------------] 62.00% | train_error: 2.74 
+
    +
    +
      [========================>---------------] 63.00% | train_error: 2.73 
+
    +
    +
      [========================>---------------] 64.00% | train_error: 2.72 
+
    +
    +
      [=========================>--------------] 65.00% | train_error: 2.70 
+
    +
    +
      [=========================>--------------] 66.00% | train_error: 2.69 
+
    +
    +
      [=========================>--------------] 67.00% | train_error: 2.68 
+
    +
    +
      [==========================>-------------] 68.00% | train_error: 2.67 
+
    +
    +
      [==========================>-------------] 69.00% | train_error: 2.65 
+
    +
    +
      [===========================>------------] 70.00% | train_error: 2.64 
+
    +
    +
      [===========================>------------] 71.00% | train_error: 2.63 
+
    +
    +
      [===========================>------------] 72.00% | train_error: 2.62 
+
    +
    +
      [============================>-----------] 73.00% | train_error: 2.60 
+
    +
    +
      [============================>-----------] 74.00% | train_error: 2.59 
+
    +
    +
      [=============================>----------] 75.00% | train_error: 2.58 
+
    +
    +
      [=============================>----------] 76.00% | train_error: 2.57 
+
    +
    +
      [=============================>----------] 77.00% | train_error: 2.55 
+
    +
    +
      [==============================>---------] 78.00% | train_error: 2.54 
+
    +
    +
      [==============================>---------] 79.00% | train_error: 2.53 
+
    +
    +
      [===============================>--------] 80.00% | train_error: 2.52 
+
    +
    +
      [===============================>--------] 81.00% | train_error: 2.51 
+
    +
    +
      [===============================>--------] 82.00% | train_error: 2.49 
+
    +
    +
      [================================>-------] 83.00% | train_error: 2.48 
+
    +
    +
      [================================>-------] 84.00% | train_error: 2.47 
+
    +
    +
      [=================================>------] 85.00% | train_error: 2.46 
+
    +
    +
      [=================================>------] 86.00% | train_error: 2.45 
+
    +
    +
      [=================================>------] 87.00% | train_error: 2.44 
+
    +
    +
      [==================================>-----] 88.00% | train_error: 2.42 
+
    +
    +
      [==================================>-----] 89.00% | train_error: 2.41 
+
    +
    +
      [===================================>----] 90.00% | train_error: 2.40 
+
    +
    +
      [===================================>----] 91.00% | train_error: 2.39 
+
    +
    +
      [===================================>----] 92.00% | train_error: 2.38 
+
    +
    +
      [====================================>---] 93.00% | train_error: 2.37 
+
    +
    +
      [====================================>---] 94.00% | train_error: 2.36 
+
    +
    +
      [=====================================>--] 95.00% | train_error: 2.34 
+
    +
    +
      [=====================================>--] 96.00% | train_error: 2.33 
+
    +
    +
      [=====================================>--] 97.00% | train_error: 2.32 
+
    +
    +
      [======================================>-] 98.00% | train_error: 2.31 
+
    +
    +
      [======================================>-] 99.00% | train_error: 2.30 
+
    +
    +
                                                                            
+
    +
    +
      [=======================================>] 100.0% | train_error: 2.30 
+
    +
    +
    +
    +

    Due to the progress bar we can see the MSE (train_error) throughout +the FFNN’s training. Note that the fit() function has some optional +parameters with defualt arguments. For example, the regularization +hyperparameter can be left ignored if not needed, and equally the FFNN +will by default run for 100 epochs. These can easily be changed, such +as for example:

    +
    +
    +
    linear_regression.reset_weights() # reset weights such that previous runs or reruns don't affect the weights
+
+scores = linear_regression.fit(X_train, t_train, scheduler, lam=1e-4, epochs=1000)
+
    +
    +
    +
    +
    Constant: Eta=0.001, Lambda=0.0001
+
+  [----------------------------------------] 0.000% | train_error: 3.69 
+
    +
    +
      [----------------------------------------] 0.1000% | train_error: 3.67 
+
    +
    +
      [----------------------------------------] 0.2000% | train_error: 3.65 
+
    +
    +
      [----------------------------------------] 0.3000% | train_error: 3.64 
+
    +
    +
      [----------------------------------------] 0.4000% | train_error: 3.62 
+
    +
    +
      [----------------------------------------] 0.5000% | train_error: 3.60 
+
    +
    +
      [----------------------------------------] 0.6000% | train_error: 3.58 
+
    +
    +
      [----------------------------------------] 0.7000% | train_error: 3.57 
+
    +
    +
      [----------------------------------------] 0.8000% | train_error: 3.55 
+
    +
    +
      [----------------------------------------] 0.9000% | train_error: 3.53 
+
    +
    +
      [----------------------------------------] 1.000% | train_error: 3.52 
+
    +
    +
      [----------------------------------------] 1.100% | train_error: 3.50 
+
    +
    +
      [----------------------------------------] 1.200% | train_error: 3.48 
+
    +
    +
      [----------------------------------------] 1.300% | train_error: 3.47 
+
    +
    +
      [----------------------------------------] 1.400% | train_error: 3.45 
+
    +
    +
      [----------------------------------------] 1.500% | train_error: 3.43 
+
    +
    +
      [----------------------------------------] 1.600% | train_error: 3.42 
+
    +
    +
      [----------------------------------------] 1.700% | train_error: 3.40 
+
    +
    +
      [----------------------------------------] 1.800% | train_error: 3.38 
+
    +
    +
      [----------------------------------------] 1.900% | train_error: 3.37 
+
    +
    +
      [----------------------------------------] 2.000% | train_error: 3.35 
+
    +
    +
      [----------------------------------------] 2.100% | train_error: 3.34 
+
    +
    +
      [----------------------------------------] 2.200% | train_error: 3.32 
+
    +
    +
      [----------------------------------------] 2.300% | train_error: 3.31 
+
    +
    +
      [----------------------------------------] 2.400% | train_error: 3.29 
+
    +
    +
      [>---------------------------------------] 2.500% | train_error: 3.27 
+
    +
    +
      [>---------------------------------------] 2.600% | train_error: 3.26 
+
    +
    +
      [>---------------------------------------] 2.700% | train_error: 3.24 
+
    +
    +
      [>---------------------------------------] 2.800% | train_error: 3.23 
+
    +
    +
      [>---------------------------------------] 2.900% | train_error: 3.21 
+
    +
    +
      [>---------------------------------------] 3.000% | train_error: 3.20 
+
    +
    +
      [>---------------------------------------] 3.100% | train_error: 3.18 
+
    +
    +
      [>---------------------------------------] 3.200% | train_error: 3.17 
+
    +
    +
      [>---------------------------------------] 3.300% | train_error: 3.15 
+
    +
    +
      [>---------------------------------------] 3.400% | train_error: 3.14 
+
    +
    +
      [>---------------------------------------] 3.500% | train_error: 3.12 
+
    +
    +
      [>---------------------------------------] 3.600% | train_error: 3.11 
+
    +
    +
      [>---------------------------------------] 3.700% | train_error: 3.09 
+
    +
    +
      [>---------------------------------------] 3.800% | train_error: 3.08 
+
    +
    +
      [>---------------------------------------] 3.900% | train_error: 3.06 
+
    +
    +
      [>---------------------------------------] 4.000% | train_error: 3.05 
+
    +
    +
      [>---------------------------------------] 4.100% | train_error: 3.03 
+
    +
    +
      [>---------------------------------------] 4.200% | train_error: 3.02 
+
    +
    +
      [>---------------------------------------] 4.300% | train_error: 3.00 
+
    +
    +
      [>---------------------------------------] 4.400% | train_error: 2.99 
+
    +
    +
      [>---------------------------------------] 4.500% | train_error: 2.98 
+
    +
    +
      [>---------------------------------------] 4.600% | train_error: 2.96 
+
    +
    +
      [>---------------------------------------] 4.700% | train_error: 2.95 
+
    +
    +
      [>---------------------------------------] 4.800% | train_error: 2.93 
+
    +
    +
      [>---------------------------------------] 4.900% | train_error: 2.92 
+
    +
    +
      [=>--------------------------------------] 5.000% | train_error: 2.91 
+
    +
    +
      [=>--------------------------------------] 5.100% | train_error: 2.89 
+
    +
    +
      [=>--------------------------------------] 5.200% | train_error: 2.88 
+
    +
    +
      [=>--------------------------------------] 5.300% | train_error: 2.86 
+
    +
    +
      [=>--------------------------------------] 5.400% | train_error: 2.85 
+
    +
    +
      [=>--------------------------------------] 5.500% | train_error: 2.84 
+
    +
    +
      [=>--------------------------------------] 5.600% | train_error: 2.82 
+
    +
    +
      [=>--------------------------------------] 5.700% | train_error: 2.81 
+
    +
    +
      [=>--------------------------------------] 5.800% | train_error: 2.80 
+
    +
    +
      [=>--------------------------------------] 5.900% | train_error: 2.78 
+
    +
    +
      [=>--------------------------------------] 6.000% | train_error: 2.77 
+
    +
    +
      [=>--------------------------------------] 6.100% | train_error: 2.76 
+
    +
    +
      [=>--------------------------------------] 6.200% | train_error: 2.74 
+
    +
    +
      [=>--------------------------------------] 6.300% | train_error: 2.73 
+
    +
    +
      [=>--------------------------------------] 6.400% | train_error: 2.72 
+
    +
    +
      [=>--------------------------------------] 6.500% | train_error: 2.70 
+
    +
    +
      [=>--------------------------------------] 6.600% | train_error: 2.69 
+
    +
    +
      [=>--------------------------------------] 6.700% | train_error: 2.68 
+
    +
    +
      [=>--------------------------------------] 6.800% | train_error: 2.67 
+
    +
    +
      [=>--------------------------------------] 6.900% | train_error: 2.65 
+
    +
    +
      [=>--------------------------------------] 7.000% | train_error: 2.64 
+
    +
    +
      [=>--------------------------------------] 7.100% | train_error: 2.63 
+
    +
    +
      [=>--------------------------------------] 7.200% | train_error: 2.62 
+
    +
    +
      [=>--------------------------------------] 7.300% | train_error: 2.60 
+
    +
    +
      [=>--------------------------------------] 7.400% | train_error: 2.59 
+
    +
    +
      [==>-------------------------------------] 7.500% | train_error: 2.58 
+
    +
    +
      [==>-------------------------------------] 7.600% | train_error: 2.57 
+
    +
    +
      [==>-------------------------------------] 7.700% | train_error: 2.55 
+
    +
    +
      [==>-------------------------------------] 7.800% | train_error: 2.54 
+
    +
    +
      [==>-------------------------------------] 7.900% | train_error: 2.53 
+
    +
    +
      [==>-------------------------------------] 8.000% | train_error: 2.52 
+
    +
    +
      [==>-------------------------------------] 8.100% | train_error: 2.51 
+
    +
    +
      [==>-------------------------------------] 8.200% | train_error: 2.49 
+
    +
    +
      [==>-------------------------------------] 8.300% | train_error: 2.48 
+
    +
    +
      [==>-------------------------------------] 8.400% | train_error: 2.47 
+
    +
    +
      [==>-------------------------------------] 8.500% | train_error: 2.46 
+
    +
    +
      [==>-------------------------------------] 8.600% | train_error: 2.45 
+
    +
    +
      [==>-------------------------------------] 8.700% | train_error: 2.44 
+
    +
    +
      [==>-------------------------------------] 8.800% | train_error: 2.42 
+
    +
    +
      [==>-------------------------------------] 8.900% | train_error: 2.41 
+
    +
    +
      [==>-------------------------------------] 9.000% | train_error: 2.40 
+
    +
    +
      [==>-------------------------------------] 9.100% | train_error: 2.39 
+
    +
    +
      [==>-------------------------------------] 9.200% | train_error: 2.38 
+
    +
    +
      [==>-------------------------------------] 9.300% | train_error: 2.37 
+
    +
    +
      [==>-------------------------------------] 9.400% | train_error: 2.36 
+
    +
    +
      [==>-------------------------------------] 9.500% | train_error: 2.34 
+
    +
    +
      [==>-------------------------------------] 9.600% | train_error: 2.33 
+
    +
    +
      [==>-------------------------------------] 9.700% | train_error: 2.32 
+
    +
    +
      [==>-------------------------------------] 9.800% | train_error: 2.31 
+
    +
    +
      [==>-------------------------------------] 9.900% | train_error: 2.30 
+
    +
    +
      [===>------------------------------------] 10.00% | train_error: 2.29 
+
    +
    +
      [===>------------------------------------] 10.10% | train_error: 2.28 
+
    +
    +
      [===>------------------------------------] 10.20% | train_error: 2.27 
+
    +
    +
      [===>------------------------------------] 10.30% | train_error: 2.26 
+
    +
    +
      [===>------------------------------------] 10.40% | train_error: 2.25 
+
    +
    +
      [===>------------------------------------] 10.50% | train_error: 2.23 
+
    +
    +
      [===>------------------------------------] 10.60% | train_error: 2.22 
+
    +
    +
      [===>------------------------------------] 10.70% | train_error: 2.21 
+
    +
    +
      [===>------------------------------------] 10.80% | train_error: 2.20 
+
    +
    +
      [===>------------------------------------] 10.90% | train_error: 2.19 
+
    +
    +
      [===>------------------------------------] 11.00% | train_error: 2.18 
+
    +
    +
      [===>------------------------------------] 11.10% | train_error: 2.17 
+
    +
    +
      [===>------------------------------------] 11.20% | train_error: 2.16 
+
    +
    +
      [===>------------------------------------] 11.30% | train_error: 2.15 
+
    +
    +
      [===>------------------------------------] 11.40% | train_error: 2.14 
+
    +
    +
      [===>------------------------------------] 11.50% | train_error: 2.13 
+
    +
    +
      [===>------------------------------------] 11.60% | train_error: 2.12 
+
    +
    +
      [===>------------------------------------] 11.70% | train_error: 2.11 
+
    +
    +
      [===>------------------------------------] 11.80% | train_error: 2.10 
+
    +
    +
      [===>------------------------------------] 11.90% | train_error: 2.09 
+
    +
    +
      [===>------------------------------------] 12.00% | train_error: 2.08 
+
    +
    +
      [===>------------------------------------] 12.10% | train_error: 2.07 
+
    +
    +
      [===>------------------------------------] 12.20% | train_error: 2.06 
+
    +
    +
      [===>------------------------------------] 12.30% | train_error: 2.05 
+
    +
    +
      [===>------------------------------------] 12.40% | train_error: 2.04 
+
    +
    +
      [====>-----------------------------------] 12.50% | train_error: 2.03 
+
    +
    +
      [====>-----------------------------------] 12.60% | train_error: 2.02 
+
    +
    +
      [====>-----------------------------------] 12.70% | train_error: 2.01 
+
    +
    +
      [====>-----------------------------------] 12.80% | train_error: 2.00 
+
    +
    +
      [====>-----------------------------------] 12.90% | train_error: 1.99 
+
    +
    +
      [====>-----------------------------------] 13.00% | train_error: 1.98 
+
    +
    +
      [====>-----------------------------------] 13.10% | train_error: 1.97 
+
    +
    +
      [====>-----------------------------------] 13.20% | train_error: 1.96 
+
    +
    +
      [====>-----------------------------------] 13.30% | train_error: 1.96 
+
    +
    +
      [====>-----------------------------------] 13.40% | train_error: 1.95 
+
    +
    +
      [====>-----------------------------------] 13.50% | train_error: 1.94 
+
    +
    +
      [====>-----------------------------------] 13.60% | train_error: 1.93 
+
    +
    +
      [====>-----------------------------------] 13.70% | train_error: 1.92 
+
    +
    +
      [====>-----------------------------------] 13.80% | train_error: 1.91 
+
    +
    +
      [====>-----------------------------------] 13.90% | train_error: 1.90 
+
    +
    +
      [====>-----------------------------------] 14.00% | train_error: 1.89 
+
    +
    +
      [====>-----------------------------------] 14.10% | train_error: 1.88 
+
    +
    +
      [====>-----------------------------------] 14.20% | train_error: 1.87 
+
    +
    +
      [====>-----------------------------------] 14.30% | train_error: 1.86 
+
    +
    +
      [====>-----------------------------------] 14.40% | train_error: 1.86 
+
    +
    +
      [====>-----------------------------------] 14.50% | train_error: 1.85 
+
    +
    +
      [====>-----------------------------------] 14.60% | train_error: 1.84 
+
    +
    +
      [====>-----------------------------------] 14.70% | train_error: 1.83 
+
    +
    +
      [====>-----------------------------------] 14.80% | train_error: 1.82 
+
    +
    +
      [====>-----------------------------------] 14.90% | train_error: 1.81 
+
    +
    +
      [=====>----------------------------------] 15.00% | train_error: 1.80 
+
    +
    +
      [=====>----------------------------------] 15.10% | train_error: 1.79 
+
    +
    +
      [=====>----------------------------------] 15.20% | train_error: 1.79 
+
    +
    +
      [=====>----------------------------------] 15.30% | train_error: 1.78 
+
    +
    +
      [=====>----------------------------------] 15.40% | train_error: 1.77 
+
    +
    +
      [=====>----------------------------------] 15.50% | train_error: 1.76 
+
    +
    +
      [=====>----------------------------------] 15.60% | train_error: 1.75 
+
    +
    +
      [=====>----------------------------------] 15.70% | train_error: 1.74 
+
    +
    +
      [=====>----------------------------------] 15.80% | train_error: 1.74 
+
    +
    +
      [=====>----------------------------------] 15.90% | train_error: 1.73 
+
    +
    +
      [=====>----------------------------------] 16.00% | train_error: 1.72 
+
    +
    +
      [=====>----------------------------------] 16.10% | train_error: 1.71 
+
    +
    +
      [=====>----------------------------------] 16.20% | train_error: 1.70 
+
    +
    +
      [=====>----------------------------------] 16.30% | train_error: 1.69 
+
    +
    +
      [=====>----------------------------------] 16.40% | train_error: 1.69 
+
    +
    +
      [=====>----------------------------------] 16.50% | train_error: 1.68 
+
    +
    +
      [=====>----------------------------------] 16.60% | train_error: 1.67 
+
    +
    +
      [=====>----------------------------------] 16.70% | train_error: 1.66 
+
    +
    +
      [=====>----------------------------------] 16.80% | train_error: 1.65 
+
    +
    +
      [=====>----------------------------------] 16.90% | train_error: 1.65 
+
    +
    +
      [=====>----------------------------------] 17.00% | train_error: 1.64 
+
    +
    +
      [=====>----------------------------------] 17.10% | train_error: 1.63 
+
    +
    +
      [=====>----------------------------------] 17.20% | train_error: 1.62 
+
    +
    +
      [=====>----------------------------------] 17.30% | train_error: 1.62 
+
    +
    +
      [=====>----------------------------------] 17.40% | train_error: 1.61 
+
    +
    +
      [======>---------------------------------] 17.50% | train_error: 1.60 
+
    +
    +
      [======>---------------------------------] 17.60% | train_error: 1.59 
+
    +
    +
      [======>---------------------------------] 17.70% | train_error: 1.59 
+
    +
    +
      [======>---------------------------------] 17.80% | train_error: 1.58 
+
    +
    +
      [======>---------------------------------] 17.90% | train_error: 1.57 
+
    +
    +
      [======>---------------------------------] 18.00% | train_error: 1.56 
+
    +
    +
      [======>---------------------------------] 18.10% | train_error: 1.56 
+
    +
    +
      [======>---------------------------------] 18.20% | train_error: 1.55 
+
    +
    +
      [======>---------------------------------] 18.30% | train_error: 1.54 
+
    +
    +
      [======>---------------------------------] 18.40% | train_error: 1.53 
+
    +
    +
      [======>---------------------------------] 18.50% | train_error: 1.53 
+
    +
    +
      [======>---------------------------------] 18.60% | train_error: 1.52 
+
    +
    +
      [======>---------------------------------] 18.70% | train_error: 1.51 
+
    +
    +
      [======>---------------------------------] 18.80% | train_error: 1.50 
+
    +
    +
      [======>---------------------------------] 18.90% | train_error: 1.50 
+
    +
    +
      [======>---------------------------------] 19.00% | train_error: 1.49 
+
    +
    +
      [======>---------------------------------] 19.10% | train_error: 1.48 
+
    +
    +
      [======>---------------------------------] 19.20% | train_error: 1.48 
+
    +
    +
      [======>---------------------------------] 19.30% | train_error: 1.47 
+
    +
    +
      [======>---------------------------------] 19.40% | train_error: 1.46 
+
    +
    +
      [======>---------------------------------] 19.50% | train_error: 1.46 
+
    +
    +
      [======>---------------------------------] 19.60% | train_error: 1.45 
+
    +
    +
      [======>---------------------------------] 19.70% | train_error: 1.44 
+
    +
    +
      [======>---------------------------------] 19.80% | train_error: 1.43 
+
    +
    +
      [======>---------------------------------] 19.90% | train_error: 1.43 
+
    +
    +
      [=======>--------------------------------] 20.00% | train_error: 1.42 
+
    +
    +
      [=======>--------------------------------] 20.10% | train_error: 1.41 
+
    +
    +
      [=======>--------------------------------] 20.20% | train_error: 1.41 
+
    +
    +
      [=======>--------------------------------] 20.30% | train_error: 1.40 
+
    +
    +
      [=======>--------------------------------] 20.40% | train_error: 1.39 
+
    +
    +
      [=======>--------------------------------] 20.50% | train_error: 1.39 
+
    +
    +
      [=======>--------------------------------] 20.60% | train_error: 1.38 
+
    +
    +
      [=======>--------------------------------] 20.70% | train_error: 1.37 
+
    +
    +
      [=======>--------------------------------] 20.80% | train_error: 1.37 
+
    +
    +
      [=======>--------------------------------] 20.90% | train_error: 1.36 
+
    +
    +
      [=======>--------------------------------] 21.00% | train_error: 1.35 
+
    +
    +
      [=======>--------------------------------] 21.10% | train_error: 1.35 
+
    +
    +
      [=======>--------------------------------] 21.20% | train_error: 1.34 
+
    +
    +
      [=======>--------------------------------] 21.30% | train_error: 1.34 
+
    +
    +
      [=======>--------------------------------] 21.40% | train_error: 1.33 
+
    +
    +
      [=======>--------------------------------] 21.50% | train_error: 1.32 
+
    +
    +
      [=======>--------------------------------] 21.60% | train_error: 1.32 
+
    +
    +
      [=======>--------------------------------] 21.70% | train_error: 1.31 
+
    +
    +
      [=======>--------------------------------] 21.80% | train_error: 1.30 
+
    +
    +
      [=======>--------------------------------] 21.90% | train_error: 1.30 
+
    +
    +
      [=======>--------------------------------] 22.00% | train_error: 1.29 
+
    +
    +
      [=======>--------------------------------] 22.10% | train_error: 1.29 
+
    +
    +
      [=======>--------------------------------] 22.20% | train_error: 1.28 
+
    +
    +
      [=======>--------------------------------] 22.30% | train_error: 1.27 
+
    +
    +
      [=======>--------------------------------] 22.40% | train_error: 1.27 
+
    +
    +
      [========>-------------------------------] 22.50% | train_error: 1.26 
+
    +
    +
      [========>-------------------------------] 22.60% | train_error: 1.26 
+
    +
    +
      [========>-------------------------------] 22.70% | train_error: 1.25 
+
    +
    +
      [========>-------------------------------] 22.80% | train_error: 1.24 
+
    +
    +
      [========>-------------------------------] 22.90% | train_error: 1.24 
+
    +
    +
      [========>-------------------------------] 23.00% | train_error: 1.23 
+
    +
    +
      [========>-------------------------------] 23.10% | train_error: 1.23 
+
    +
    +
      [========>-------------------------------] 23.20% | train_error: 1.22 
+
    +
    +
      [========>-------------------------------] 23.30% | train_error: 1.21 
+
    +
    +
      [========>-------------------------------] 23.40% | train_error: 1.21 
+
    +
    +
      [========>-------------------------------] 23.50% | train_error: 1.20 
+
    +
    +
      [========>-------------------------------] 23.60% | train_error: 1.20 
+
    +
    +
      [========>-------------------------------] 23.70% | train_error: 1.19 
+
    +
    +
      [========>-------------------------------] 23.80% | train_error: 1.19 
+
    +
    +
      [========>-------------------------------] 23.90% | train_error: 1.18 
+
    +
    +
      [========>-------------------------------] 24.00% | train_error: 1.17 
+
    +
    +
      [========>-------------------------------] 24.10% | train_error: 1.17 
+
    +
    +
      [========>-------------------------------] 24.20% | train_error: 1.16 
+
    +
    +
      [========>-------------------------------] 24.30% | train_error: 1.16 
+
    +
    +
      [========>-------------------------------] 24.40% | train_error: 1.15 
+
    +
    +
      [========>-------------------------------] 24.50% | train_error: 1.15 
+
    +
    +
      [========>-------------------------------] 24.60% | train_error: 1.14 
+
    +
    +
      [========>-------------------------------] 24.70% | train_error: 1.14 
+
    +
    +
      [========>-------------------------------] 24.80% | train_error: 1.13 
+
    +
    +
      [========>-------------------------------] 24.90% | train_error: 1.13 
+
    +
    +
      [=========>------------------------------] 25.00% | train_error: 1.12 
+
    +
    +
      [=========>------------------------------] 25.10% | train_error: 1.11 
+
    +
    +
      [=========>------------------------------] 25.20% | train_error: 1.11 
+
    +
    +
      [=========>------------------------------] 25.30% | train_error: 1.10 
+
    +
    +
      [=========>------------------------------] 25.40% | train_error: 1.10 
+
    +
    +
      [=========>------------------------------] 25.50% | train_error: 1.09 
+
    +
    +
      [=========>------------------------------] 25.60% | train_error: 1.09 
+
    +
    +
      [=========>------------------------------] 25.70% | train_error: 1.08 
+
    +
    +
      [=========>------------------------------] 25.80% | train_error: 1.08 
+
    +
    +
      [=========>------------------------------] 25.90% | train_error: 1.07 
+
    +
    +
      [=========>------------------------------] 26.00% | train_error: 1.07 
+
    +
    +
      [=========>------------------------------] 26.10% | train_error: 1.06 
+
    +
    +
      [=========>------------------------------] 26.20% | train_error: 1.06 
+
    +
    +
      [=========>------------------------------] 26.30% | train_error: 1.05 
+
    +
    +
      [=========>------------------------------] 26.40% | train_error: 1.05 
+
    +
    +
      [=========>------------------------------] 26.50% | train_error: 1.04 
+
    +
    +
      [=========>------------------------------] 26.60% | train_error: 1.04 
+
    +
    +
      [=========>------------------------------] 26.70% | train_error: 1.03 
+
    +
    +
      [=========>------------------------------] 26.80% | train_error: 1.03 
+
    +
    +
      [=========>------------------------------] 26.90% | train_error: 1.02 
+
    +
    +
      [=========>------------------------------] 27.00% | train_error: 1.02 
+
    +
    +
      [=========>------------------------------] 27.10% | train_error: 1.01 
+
    +
    +
      [=========>------------------------------] 27.20% | train_error: 1.01 
+
    +
    +
      [=========>------------------------------] 27.30% | train_error: 1.00 
+
    +
    +
      [=========>------------------------------] 27.40% | train_error: 0.999 
+
    +
    +
      [==========>-----------------------------] 27.50% | train_error: 0.994 
+
    +
    +
      [==========>-----------------------------] 27.60% | train_error: 0.990 
+
    +
    +
      [==========>-----------------------------] 27.70% | train_error: 0.985 
+
    +
    +
      [==========>-----------------------------] 27.80% | train_error: 0.980 
+
    +
    +
      [==========>-----------------------------] 27.90% | train_error: 0.976 
+
    +
    +
      [==========>-----------------------------] 28.00% | train_error: 0.971 
+
    +
    +
      [==========>-----------------------------] 28.10% | train_error: 0.966 
+
    +
    +
      [==========>-----------------------------] 28.20% | train_error: 0.962 
+
    +
    +
      [==========>-----------------------------] 28.30% | train_error: 0.957 
+
    +
    +
      [==========>-----------------------------] 28.40% | train_error: 0.953 
+
    +
    +
      [==========>-----------------------------] 28.50% | train_error: 0.948 
+
    +
    +
      [==========>-----------------------------] 28.60% | train_error: 0.944 
+
    +
    +
      [==========>-----------------------------] 28.70% | train_error: 0.939 
+
    +
    +
      [==========>-----------------------------] 28.80% | train_error: 0.935 
+
    +
    +
      [==========>-----------------------------] 28.90% | train_error: 0.930 
+
    +
    +
      [==========>-----------------------------] 29.00% | train_error: 0.926 
+
    +
    +
      [==========>-----------------------------] 29.10% | train_error: 0.922 
+
    +
    +
      [==========>-----------------------------] 29.20% | train_error: 0.917 
+
    +
    +
      [==========>-----------------------------] 29.30% | train_error: 0.913 
+
    +
    +
      [==========>-----------------------------] 29.40% | train_error: 0.909 
+
    +
    +
      [==========>-----------------------------] 29.50% | train_error: 0.904 
+
    +
    +
      [==========>-----------------------------] 29.60% | train_error: 0.900 
+
    +
    +
      [==========>-----------------------------] 29.70% | train_error: 0.896 
+
    +
    +
      [==========>-----------------------------] 29.80% | train_error: 0.891 
+
    +
    +
      [==========>-----------------------------] 29.90% | train_error: 0.887 
+
    +
    +
      [===========>----------------------------] 30.00% | train_error: 0.883 
+
    +
    +
      [===========>----------------------------] 30.10% | train_error: 0.879 
+
    +
    +
      [===========>----------------------------] 30.20% | train_error: 0.875 
+
    +
    +
      [===========>----------------------------] 30.30% | train_error: 0.870 
+
    +
    +
      [===========>----------------------------] 30.40% | train_error: 0.866 
+
    +
    +
      [===========>----------------------------] 30.50% | train_error: 0.862 
+
    +
    +
      [===========>----------------------------] 30.60% | train_error: 0.858 
+
    +
    +
      [===========>----------------------------] 30.70% | train_error: 0.854 
+
    +
    +
      [===========>----------------------------] 30.80% | train_error: 0.850 
+
    +
    +
      [===========>----------------------------] 30.90% | train_error: 0.846 
+
    +
    +
      [===========>----------------------------] 31.00% | train_error: 0.842 
+
    +
    +
      [===========>----------------------------] 31.10% | train_error: 0.838 
+
    +
    +
      [===========>----------------------------] 31.20% | train_error: 0.834 
+
    +
    +
      [===========>----------------------------] 31.30% | train_error: 0.830 
+
    +
    +
      [===========>----------------------------] 31.40% | train_error: 0.826 
+
    +
    +
      [===========>----------------------------] 31.50% | train_error: 0.822 
+
    +
    +
      [===========>----------------------------] 31.60% | train_error: 0.818 
+
    +
    +
      [===========>----------------------------] 31.70% | train_error: 0.814 
+
    +
    +
      [===========>----------------------------] 31.80% | train_error: 0.811 
+
    +
    +
      [===========>----------------------------] 31.90% | train_error: 0.807 
+
    +
    +
      [===========>----------------------------] 32.00% | train_error: 0.803 
+
    +
    +
      [===========>----------------------------] 32.10% | train_error: 0.799 
+
    +
    +
      [===========>----------------------------] 32.20% | train_error: 0.795 
+
    +
    +
      [===========>----------------------------] 32.30% | train_error: 0.792 
+
    +
    +
      [===========>----------------------------] 32.40% | train_error: 0.788 
+
    +
    +
      [============>---------------------------] 32.50% | train_error: 0.784 
+
    +
    +
      [============>---------------------------] 32.60% | train_error: 0.780 
+
    +
    +
      [============>---------------------------] 32.70% | train_error: 0.777 
+
    +
    +
      [============>---------------------------] 32.80% | train_error: 0.773 
+
    +
    +
      [============>---------------------------] 32.90% | train_error: 0.769 
+
    +
    +
      [============>---------------------------] 33.00% | train_error: 0.766 
+
    +
    +
      [============>---------------------------] 33.10% | train_error: 0.762 
+
    +
    +
      [============>---------------------------] 33.20% | train_error: 0.759 
+
    +
    +
      [============>---------------------------] 33.30% | train_error: 0.755 
+
    +
    +
      [============>---------------------------] 33.40% | train_error: 0.751 
+
    +
    +
      [============>---------------------------] 33.50% | train_error: 0.748 
+
    +
    +
      [============>---------------------------] 33.60% | train_error: 0.744 
+
    +
    +
      [============>---------------------------] 33.70% | train_error: 0.741 
+
    +
    +
      [============>---------------------------] 33.80% | train_error: 0.737 
+
    +
    +
      [============>---------------------------] 33.90% | train_error: 0.734 
+
    +
    +
      [============>---------------------------] 34.00% | train_error: 0.730 
+
    +
    +
      [============>---------------------------] 34.10% | train_error: 0.727 
+
    +
    +
      [============>---------------------------] 34.20% | train_error: 0.723 
+
    +
    +
      [============>---------------------------] 34.30% | train_error: 0.720 
+
    +
    +
      [============>---------------------------] 34.40% | train_error: 0.717 
+
    +
    +
      [============>---------------------------] 34.50% | train_error: 0.713 
+
    +
    +
      [============>---------------------------] 34.60% | train_error: 0.710 
+
    +
    +
      [============>---------------------------] 34.70% | train_error: 0.706 
+
    +
    +
      [============>---------------------------] 34.80% | train_error: 0.703 
+
    +
    +
      [============>---------------------------] 34.90% | train_error: 0.700 
+
    +
    +
      [=============>--------------------------] 35.00% | train_error: 0.696 
+
    +
    +
      [=============>--------------------------] 35.10% | train_error: 0.693 
+
    +
    +
      [=============>--------------------------] 35.20% | train_error: 0.690 
+
    +
    +
      [=============>--------------------------] 35.30% | train_error: 0.687 
+
    +
    +
      [=============>--------------------------] 35.40% | train_error: 0.683 
+
    +
    +
      [=============>--------------------------] 35.50% | train_error: 0.680 
+
    +
    +
      [=============>--------------------------] 35.60% | train_error: 0.677 
+
    +
    +
      [=============>--------------------------] 35.70% | train_error: 0.674 
+
    +
    +
      [=============>--------------------------] 35.80% | train_error: 0.670 
+
    +
    +
      [=============>--------------------------] 35.90% | train_error: 0.667 
+
    +
    +
      [=============>--------------------------] 36.00% | train_error: 0.664 
+
    +
    +
      [=============>--------------------------] 36.10% | train_error: 0.661 
+
    +
    +
      [=============>--------------------------] 36.20% | train_error: 0.658 
+
    +
    +
      [=============>--------------------------] 36.30% | train_error: 0.655 
+
    +
    +
      [=============>--------------------------] 36.40% | train_error: 0.652 
+
    +
    +
      [=============>--------------------------] 36.50% | train_error: 0.649 
+
    +
    +
      [=============>--------------------------] 36.60% | train_error: 0.646 
+
    +
    +
      [=============>--------------------------] 36.70% | train_error: 0.642 
+
    +
    +
      [=============>--------------------------] 36.80% | train_error: 0.639 
+
    +
    +
      [=============>--------------------------] 36.90% | train_error: 0.636 
+
    +
    +
      [=============>--------------------------] 37.00% | train_error: 0.633 
+
    +
    +
      [=============>--------------------------] 37.10% | train_error: 0.630 
+
    +
    +
      [=============>--------------------------] 37.20% | train_error: 0.627 
+
    +
    +
      [=============>--------------------------] 37.30% | train_error: 0.624 
+
    +
    +
      [=============>--------------------------] 37.40% | train_error: 0.622 
+
    +
    +
      [==============>-------------------------] 37.50% | train_error: 0.619 
+
    +
    +
      [==============>-------------------------] 37.60% | train_error: 0.616 
+
    +
    +
      [==============>-------------------------] 37.70% | train_error: 0.613 
+
    +
    +
      [==============>-------------------------] 37.80% | train_error: 0.610 
+
    +
    +
      [==============>-------------------------] 37.90% | train_error: 0.607 
+
    +
    +
      [==============>-------------------------] 38.00% | train_error: 0.604 
+
    +
    +
      [==============>-------------------------] 38.10% | train_error: 0.601 
+
    +
    +
      [==============>-------------------------] 38.20% | train_error: 0.598 
+
    +
    +
      [==============>-------------------------] 38.30% | train_error: 0.596 
+
    +
    +
      [==============>-------------------------] 38.40% | train_error: 0.593 
+
    +
    +
      [==============>-------------------------] 38.50% | train_error: 0.590 
+
    +
    +
      [==============>-------------------------] 38.60% | train_error: 0.587 
+
    +
    +
      [==============>-------------------------] 38.70% | train_error: 0.584 
+
    +
    +
      [==============>-------------------------] 38.80% | train_error: 0.582 
+
    +
    +
      [==============>-------------------------] 38.90% | train_error: 0.579 
+
    +
    +
      [==============>-------------------------] 39.00% | train_error: 0.576 
+
    +
    +
      [==============>-------------------------] 39.10% | train_error: 0.573 
+
    +
    +
      [==============>-------------------------] 39.20% | train_error: 0.571 
+
    +
    +
      [==============>-------------------------] 39.30% | train_error: 0.568 
+
    +
    +
      [==============>-------------------------] 39.40% | train_error: 0.565 
+
    +
    +
      [==============>-------------------------] 39.50% | train_error: 0.563 
+
    +
    +
      [==============>-------------------------] 39.60% | train_error: 0.560 
+
    +
    +
      [==============>-------------------------] 39.70% | train_error: 0.557 
+
    +
    +
      [==============>-------------------------] 39.80% | train_error: 0.555 
+
    +
    +
      [==============>-------------------------] 39.90% | train_error: 0.552 
+
    +
    +
      [===============>------------------------] 40.00% | train_error: 0.549 
+
    +
    +
      [===============>------------------------] 40.10% | train_error: 0.547 
+
    +
    +
      [===============>------------------------] 40.20% | train_error: 0.544 
+
    +
    +
      [===============>------------------------] 40.30% | train_error: 0.542 
+
    +
    +
      [===============>------------------------] 40.40% | train_error: 0.539 
+
    +
    +
      [===============>------------------------] 40.50% | train_error: 0.537 
+
    +
    +
      [===============>------------------------] 40.60% | train_error: 0.534 
+
    +
    +
      [===============>------------------------] 40.70% | train_error: 0.532 
+
    +
    +
      [===============>------------------------] 40.80% | train_error: 0.529 
+
    +
    +
      [===============>------------------------] 40.90% | train_error: 0.527 
+
    +
    +
      [===============>------------------------] 41.00% | train_error: 0.524 
+
    +
    +
      [===============>------------------------] 41.10% | train_error: 0.522 
+
    +
    +
      [===============>------------------------] 41.20% | train_error: 0.519 
+
    +
    +
      [===============>------------------------] 41.30% | train_error: 0.517 
+
    +
    +
      [===============>------------------------] 41.40% | train_error: 0.514 
+
    +
    +
      [===============>------------------------] 41.50% | train_error: 0.512 
+
    +
    +
      [===============>------------------------] 41.60% | train_error: 0.509 
+
    +
    +
      [===============>------------------------] 41.70% | train_error: 0.507 
+
    +
    +
      [===============>------------------------] 41.80% | train_error: 0.505 
+
    +
    +
      [===============>------------------------] 41.90% | train_error: 0.502 
+
    +
    +
      [===============>------------------------] 42.00% | train_error: 0.500 
+
    +
    +
      [===============>------------------------] 42.10% | train_error: 0.498 
+
    +
    +
      [===============>------------------------] 42.20% | train_error: 0.495 
+
    +
    +
      [===============>------------------------] 42.30% | train_error: 0.493 
+
    +
    +
      [===============>------------------------] 42.40% | train_error: 0.491 
+
    +
    +
      [================>-----------------------] 42.50% | train_error: 0.488 
+
    +
    +
      [================>-----------------------] 42.60% | train_error: 0.486 
+
    +
    +
      [================>-----------------------] 42.70% | train_error: 0.484 
+
    +
    +
      [================>-----------------------] 42.80% | train_error: 0.481 
+
    +
    +
      [================>-----------------------] 42.90% | train_error: 0.479 
+
    +
    +
      [================>-----------------------] 43.00% | train_error: 0.477 
+
    +
    +
      [================>-----------------------] 43.10% | train_error: 0.475 
+
    +
    +
      [================>-----------------------] 43.20% | train_error: 0.472 
+
    +
    +
      [================>-----------------------] 43.30% | train_error: 0.470 
+
    +
    +
      [================>-----------------------] 43.40% | train_error: 0.468 
+
    +
    +
      [================>-----------------------] 43.50% | train_error: 0.466 
+
    +
    +
      [================>-----------------------] 43.60% | train_error: 0.463 
+
    +
    +
      [================>-----------------------] 43.70% | train_error: 0.461 
+
    +
    +
      [================>-----------------------] 43.80% | train_error: 0.459 
+
    +
    +
      [================>-----------------------] 43.90% | train_error: 0.457 
+
    +
    +
      [================>-----------------------] 44.00% | train_error: 0.455 
+
    +
    +
      [================>-----------------------] 44.10% | train_error: 0.453 
+
    +
    +
      [================>-----------------------] 44.20% | train_error: 0.451 
+
    +
    +
      [================>-----------------------] 44.30% | train_error: 0.448 
+
    +
    +
      [================>-----------------------] 44.40% | train_error: 0.446 
+
    +
    +
      [================>-----------------------] 44.50% | train_error: 0.444 
+
    +
    +
      [================>-----------------------] 44.60% | train_error: 0.442 
+
    +
    +
      [================>-----------------------] 44.70% | train_error: 0.440 
+
    +
    +
      [================>-----------------------] 44.80% | train_error: 0.438 
+
    +
    +
      [================>-----------------------] 44.90% | train_error: 0.436 
+
    +
    +
      [=================>----------------------] 45.00% | train_error: 0.434 
+
    +
    +
      [=================>----------------------] 45.10% | train_error: 0.432 
+
    +
    +
      [=================>----------------------] 45.20% | train_error: 0.430 
+
    +
    +
      [=================>----------------------] 45.30% | train_error: 0.428 
+
    +
    +
      [=================>----------------------] 45.40% | train_error: 0.426 
+
    +
    +
      [=================>----------------------] 45.50% | train_error: 0.424 
+
    +
    +
      [=================>----------------------] 45.60% | train_error: 0.422 
+
    +
    +
      [=================>----------------------] 45.70% | train_error: 0.420 
+
    +
    +
      [=================>----------------------] 45.80% | train_error: 0.418 
+
    +
    +
      [=================>----------------------] 45.90% | train_error: 0.416 
+
    +
    +
      [=================>----------------------] 46.00% | train_error: 0.414 
+
    +
    +
      [=================>----------------------] 46.10% | train_error: 0.412 
+
    +
    +
      [=================>----------------------] 46.20% | train_error: 0.410 
+
    +
    +
      [=================>----------------------] 46.30% | train_error: 0.408 
+
    +
    +
      [=================>----------------------] 46.40% | train_error: 0.406 
+
    +
    +
      [=================>----------------------] 46.50% | train_error: 0.404 
+
    +
    +
      [=================>----------------------] 46.60% | train_error: 0.402 
+
    +
    +
      [=================>----------------------] 46.70% | train_error: 0.400 
+
    +
    +
      [=================>----------------------] 46.80% | train_error: 0.398 
+
    +
    +
      [=================>----------------------] 46.90% | train_error: 0.397 
+
    +
    +
      [=================>----------------------] 47.00% | train_error: 0.395 
+
    +
    +
      [=================>----------------------] 47.10% | train_error: 0.393 
+
    +
    +
      [=================>----------------------] 47.20% | train_error: 0.391 
+
    +
    +
      [=================>----------------------] 47.30% | train_error: 0.389 
+
    +
    +
      [=================>----------------------] 47.40% | train_error: 0.387 
+
    +
    +
      [==================>---------------------] 47.50% | train_error: 0.386 
+
    +
    +
      [==================>---------------------] 47.60% | train_error: 0.384 
+
    +
    +
      [==================>---------------------] 47.70% | train_error: 0.382 
+
    +
    +
      [==================>---------------------] 47.80% | train_error: 0.380 
+
    +
    +
      [==================>---------------------] 47.90% | train_error: 0.378 
+
    +
    +
      [==================>---------------------] 48.00% | train_error: 0.377 
+
    +
    +
      [==================>---------------------] 48.10% | train_error: 0.375 
+
    +
    +
      [==================>---------------------] 48.20% | train_error: 0.373 
+
    +
    +
      [==================>---------------------] 48.30% | train_error: 0.371 
+
    +
    +
      [==================>---------------------] 48.40% | train_error: 0.370 
+
    +
    +
      [==================>---------------------] 48.50% | train_error: 0.368 
+
    +
    +
      [==================>---------------------] 48.60% | train_error: 0.366 
+
    +
    +
      [==================>---------------------] 48.70% | train_error: 0.364 
+
    +
    +
      [==================>---------------------] 48.80% | train_error: 0.363 
+
    +
    +
      [==================>---------------------] 48.90% | train_error: 0.361 
+
    +
    +
      [==================>---------------------] 49.00% | train_error: 0.359 
+
    +
    +
      [==================>---------------------] 49.10% | train_error: 0.358 
+
    +
    +
      [==================>---------------------] 49.20% | train_error: 0.356 
+
    +
    +
      [==================>---------------------] 49.30% | train_error: 0.354 
+
    +
    +
      [==================>---------------------] 49.40% | train_error: 0.353 
+
    +
    +
      [==================>---------------------] 49.50% | train_error: 0.351 
+
    +
    +
      [==================>---------------------] 49.60% | train_error: 0.349 
+
    +
    +
      [==================>---------------------] 49.70% | train_error: 0.348 
+
    +
    +
      [==================>---------------------] 49.80% | train_error: 0.346 
+
    +
    +
      [==================>---------------------] 49.90% | train_error: 0.344 
+
    +
    +
      [===================>--------------------] 50.00% | train_error: 0.343 
+
    +
    +
      [===================>--------------------] 50.10% | train_error: 0.341 
+
    +
    +
      [===================>--------------------] 50.20% | train_error: 0.339 
+
    +
    +
      [===================>--------------------] 50.30% | train_error: 0.338 
+
    +
    +
      [===================>--------------------] 50.40% | train_error: 0.336 
+
    +
    +
      [===================>--------------------] 50.50% | train_error: 0.335 
+
    +
    +
      [===================>--------------------] 50.60% | train_error: 0.333 
+
    +
    +
      [===================>--------------------] 50.70% | train_error: 0.332 
+
    +
    +
      [===================>--------------------] 50.80% | train_error: 0.330 
+
    +
    +
      [===================>--------------------] 50.90% | train_error: 0.328 
+
    +
    +
      [===================>--------------------] 51.00% | train_error: 0.327 
+
    +
    +
      [===================>--------------------] 51.10% | train_error: 0.325 
+
    +
    +
      [===================>--------------------] 51.20% | train_error: 0.324 
+
    +
    +
      [===================>--------------------] 51.30% | train_error: 0.322 
+
    +
    +
      [===================>--------------------] 51.40% | train_error: 0.321 
+
    +
    +
      [===================>--------------------] 51.50% | train_error: 0.319 
+
    +
    +
      [===================>--------------------] 51.60% | train_error: 0.318 
+
    +
    +
      [===================>--------------------] 51.70% | train_error: 0.316 
+
    +
    +
      [===================>--------------------] 51.80% | train_error: 0.315 
+
    +
    +
      [===================>--------------------] 51.90% | train_error: 0.313 
+
    +
    +
      [===================>--------------------] 52.00% | train_error: 0.312 
+
    +
    +
      [===================>--------------------] 52.10% | train_error: 0.310 
+
    +
    +
      [===================>--------------------] 52.20% | train_error: 0.309 
+
    +
    +
      [===================>--------------------] 52.30% | train_error: 0.308 
+
    +
    +
      [===================>--------------------] 52.40% | train_error: 0.306 
+
    +
    +
      [====================>-------------------] 52.50% | train_error: 0.305 
+
    +
    +
      [====================>-------------------] 52.60% | train_error: 0.303 
+
    +
    +
      [====================>-------------------] 52.70% | train_error: 0.302 
+
    +
    +
      [====================>-------------------] 52.80% | train_error: 0.300 
+
    +
    +
      [====================>-------------------] 52.90% | train_error: 0.299 
+
    +
    +
      [====================>-------------------] 53.00% | train_error: 0.298 
+
    +
    +
      [====================>-------------------] 53.10% | train_error: 0.296 
+
    +
    +
      [====================>-------------------] 53.20% | train_error: 0.295 
+
    +
    +
      [====================>-------------------] 53.30% | train_error: 0.293 
+
    +
    +
      [====================>-------------------] 53.40% | train_error: 0.292 
+
    +
    +
      [====================>-------------------] 53.50% | train_error: 0.291 
+
    +
    +
      [====================>-------------------] 53.60% | train_error: 0.289 
+
    +
    +
      [====================>-------------------] 53.70% | train_error: 0.288 
+
    +
    +
      [====================>-------------------] 53.80% | train_error: 0.287 
+
    +
    +
      [====================>-------------------] 53.90% | train_error: 0.285 
+
    +
    +
      [====================>-------------------] 54.00% | train_error: 0.284 
+
    +
    +
      [====================>-------------------] 54.10% | train_error: 0.283 
+
    +
    +
      [====================>-------------------] 54.20% | train_error: 0.281 
+
    +
    +
      [====================>-------------------] 54.30% | train_error: 0.280 
+
    +
    +
      [====================>-------------------] 54.40% | train_error: 0.279 
+
    +
    +
      [====================>-------------------] 54.50% | train_error: 0.277 
+
    +
    +
      [====================>-------------------] 54.60% | train_error: 0.276 
+
    +
    +
      [====================>-------------------] 54.70% | train_error: 0.275 
+
    +
    +
      [====================>-------------------] 54.80% | train_error: 0.273 
+
    +
    +
      [====================>-------------------] 54.90% | train_error: 0.272 
+
    +
    +
      [=====================>------------------] 55.00% | train_error: 0.271 
+
    +
    +
      [=====================>------------------] 55.10% | train_error: 0.270 
+
    +
    +
      [=====================>------------------] 55.20% | train_error: 0.268 
+
    +
    +
      [=====================>------------------] 55.30% | train_error: 0.267 
+
    +
    +
      [=====================>------------------] 55.40% | train_error: 0.266 
+
    +
    +
      [=====================>------------------] 55.50% | train_error: 0.265 
+
    +
    +
      [=====================>------------------] 55.60% | train_error: 0.263 
+
    +
    +
      [=====================>------------------] 55.70% | train_error: 0.262 
+
    +
    +
      [=====================>------------------] 55.80% | train_error: 0.261 
+
    +
    +
      [=====================>------------------] 55.90% | train_error: 0.260 
+
    +
    +
      [=====================>------------------] 56.00% | train_error: 0.259 
+
    +
    +
      [=====================>------------------] 56.10% | train_error: 0.257 
+
    +
    +
      [=====================>------------------] 56.20% | train_error: 0.256 
+
    +
    +
      [=====================>------------------] 56.30% | train_error: 0.255 
+
    +
    +
      [=====================>------------------] 56.40% | train_error: 0.254 
+
    +
    +
      [=====================>------------------] 56.50% | train_error: 0.253 
+
    +
    +
      [=====================>------------------] 56.60% | train_error: 0.251 
+
    +
    +
      [=====================>------------------] 56.70% | train_error: 0.250 
+
    +
    +
      [=====================>------------------] 56.80% | train_error: 0.249 
+
    +
    +
      [=====================>------------------] 56.90% | train_error: 0.248 
+
    +
    +
      [=====================>------------------] 57.00% | train_error: 0.247 
+
    +
    +
      [=====================>------------------] 57.10% | train_error: 0.246 
+
    +
    +
      [=====================>------------------] 57.20% | train_error: 0.244 
+
    +
    +
      [=====================>------------------] 57.30% | train_error: 0.243 
+
    +
    +
      [=====================>------------------] 57.40% | train_error: 0.242 
+
    +
    +
      [======================>-----------------] 57.50% | train_error: 0.241 
+
    +
    +
      [======================>-----------------] 57.60% | train_error: 0.240 
+
    +
    +
      [======================>-----------------] 57.70% | train_error: 0.239 
+
    +
    +
      [======================>-----------------] 57.80% | train_error: 0.238 
+
    +
    +
      [======================>-----------------] 57.90% | train_error: 0.237 
+
    +
    +
      [======================>-----------------] 58.00% | train_error: 0.235 
+
    +
    +
      [======================>-----------------] 58.10% | train_error: 0.234 
+
    +
    +
      [======================>-----------------] 58.20% | train_error: 0.233 
+
    +
    +
      [======================>-----------------] 58.30% | train_error: 0.232 
+
    +
    +
      [======================>-----------------] 58.40% | train_error: 0.231 
+
    +
    +
      [======================>-----------------] 58.50% | train_error: 0.230 
+
    +
    +
      [======================>-----------------] 58.60% | train_error: 0.229 
+
    +
    +
      [======================>-----------------] 58.70% | train_error: 0.228 
+
    +
    +
      [======================>-----------------] 58.80% | train_error: 0.227 
+
    +
    +
      [======================>-----------------] 58.90% | train_error: 0.226 
+
    +
    +
      [======================>-----------------] 59.00% | train_error: 0.225 
+
    +
    +
      [======================>-----------------] 59.10% | train_error: 0.224 
+
    +
    +
      [======================>-----------------] 59.20% | train_error: 0.223 
+
    +
    +
      [======================>-----------------] 59.30% | train_error: 0.222 
+
    +
    +
      [======================>-----------------] 59.40% | train_error: 0.221 
+
    +
    +
      [======================>-----------------] 59.50% | train_error: 0.219 
+
    +
    +
      [======================>-----------------] 59.60% | train_error: 0.218 
+
    +
    +
      [======================>-----------------] 59.70% | train_error: 0.217 
+
    +
    +
      [======================>-----------------] 59.80% | train_error: 0.216 
+
    +
    +
      [======================>-----------------] 59.90% | train_error: 0.215 
+
    +
    +
      [=======================>----------------] 60.00% | train_error: 0.214 
+
    +
    +
      [=======================>----------------] 60.10% | train_error: 0.213 
+
    +
    +
      [=======================>----------------] 60.20% | train_error: 0.212 
+
    +
    +
      [=======================>----------------] 60.30% | train_error: 0.211 
+
    +
    +
      [=======================>----------------] 60.40% | train_error: 0.210 
+
    +
    +
      [=======================>----------------] 60.50% | train_error: 0.209 
+
    +
    +
      [=======================>----------------] 60.60% | train_error: 0.209 
+
    +
    +
      [=======================>----------------] 60.70% | train_error: 0.208 
+
    +
    +
      [=======================>----------------] 60.80% | train_error: 0.207 
+
    +
    +
      [=======================>----------------] 60.90% | train_error: 0.206 
+
    +
    +
      [=======================>----------------] 61.00% | train_error: 0.205 
+
    +
    +
      [=======================>----------------] 61.10% | train_error: 0.204 
+
    +
    +
      [=======================>----------------] 61.20% | train_error: 0.203 
+
    +
    +
      [=======================>----------------] 61.30% | train_error: 0.202 
+
    +
    +
      [=======================>----------------] 61.40% | train_error: 0.201 
+
    +
    +
      [=======================>----------------] 61.50% | train_error: 0.200 
+
    +
    +
      [=======================>----------------] 61.60% | train_error: 0.199 
+
    +
    +
      [=======================>----------------] 61.70% | train_error: 0.198 
+
    +
    +
      [=======================>----------------] 61.80% | train_error: 0.197 
+
    +
    +
      [=======================>----------------] 61.90% | train_error: 0.196 
+
    +
    +
      [=======================>----------------] 62.00% | train_error: 0.195 
+
    +
    +
      [=======================>----------------] 62.10% | train_error: 0.194 
+
    +
    +
      [=======================>----------------] 62.20% | train_error: 0.194 
+
    +
    +
      [=======================>----------------] 62.30% | train_error: 0.193 
+
    +
    +
      [=======================>----------------] 62.40% | train_error: 0.192 
+
    +
    +
      [========================>---------------] 62.50% | train_error: 0.191 
+
    +
    +
      [========================>---------------] 62.60% | train_error: 0.190 
+
    +
    +
      [========================>---------------] 62.70% | train_error: 0.189 
+
    +
    +
      [========================>---------------] 62.80% | train_error: 0.188 
+
    +
    +
      [========================>---------------] 62.90% | train_error: 0.187 
+
    +
    +
      [========================>---------------] 63.00% | train_error: 0.186 
+
    +
    +
      [========================>---------------] 63.10% | train_error: 0.186 
+
    +
    +
      [========================>---------------] 63.20% | train_error: 0.185 
+
    +
    +
      [========================>---------------] 63.30% | train_error: 0.184 
+
    +
    +
      [========================>---------------] 63.40% | train_error: 0.183 
+
    +
    +
      [========================>---------------] 63.50% | train_error: 0.182 
+
    +
    +
      [========================>---------------] 63.60% | train_error: 0.181 
+
    +
    +
      [========================>---------------] 63.70% | train_error: 0.180 
+
    +
    +
      [========================>---------------] 63.80% | train_error: 0.180 
+
    +
    +
      [========================>---------------] 63.90% | train_error: 0.179 
+
    +
    +
      [========================>---------------] 64.00% | train_error: 0.178 
+
    +
    +
      [========================>---------------] 64.10% | train_error: 0.177 
+
    +
    +
      [========================>---------------] 64.20% | train_error: 0.176 
+
    +
    +
      [========================>---------------] 64.30% | train_error: 0.176 
+
    +
    +
      [========================>---------------] 64.40% | train_error: 0.175 
+
    +
    +
      [========================>---------------] 64.50% | train_error: 0.174 
+
    +
    +
      [========================>---------------] 64.60% | train_error: 0.173 
+
    +
    +
      [========================>---------------] 64.70% | train_error: 0.172 
+
    +
    +
      [========================>---------------] 64.80% | train_error: 0.171 
+
    +
    +
      [========================>---------------] 64.90% | train_error: 0.171 
+
    +
    +
      [=========================>--------------] 65.00% | train_error: 0.170 
+
    +
    +
      [=========================>--------------] 65.10% | train_error: 0.169 
+
    +
    +
      [=========================>--------------] 65.20% | train_error: 0.168 
+
    +
    +
      [=========================>--------------] 65.30% | train_error: 0.168 
+
    +
    +
      [=========================>--------------] 65.40% | train_error: 0.167 
+
    +
    +
      [=========================>--------------] 65.50% | train_error: 0.166 
+
    +
    +
      [=========================>--------------] 65.60% | train_error: 0.165 
+
    +
    +
      [=========================>--------------] 65.70% | train_error: 0.164 
+
    +
    +
      [=========================>--------------] 65.80% | train_error: 0.164 
+
    +
    +
      [=========================>--------------] 65.90% | train_error: 0.163 
+
    +
    +
      [=========================>--------------] 66.00% | train_error: 0.162 
+
    +
    +
      [=========================>--------------] 66.10% | train_error: 0.161 
+
    +
    +
      [=========================>--------------] 66.20% | train_error: 0.161 
+
    +
    +
      [=========================>--------------] 66.30% | train_error: 0.160 
+
    +
    +
      [=========================>--------------] 66.40% | train_error: 0.159 
+
    +
    +
      [=========================>--------------] 66.50% | train_error: 0.158 
+
    +
    +
      [=========================>--------------] 66.60% | train_error: 0.158 
+
    +
    +
      [=========================>--------------] 66.70% | train_error: 0.157 
+
    +
    +
      [=========================>--------------] 66.80% | train_error: 0.156 
+
    +
    +
      [=========================>--------------] 66.90% | train_error: 0.156 
+
    +
    +
      [=========================>--------------] 67.00% | train_error: 0.155 
+
    +
    +
      [=========================>--------------] 67.10% | train_error: 0.154 
+
    +
    +
      [=========================>--------------] 67.20% | train_error: 0.153 
+
    +
    +
      [=========================>--------------] 67.30% | train_error: 0.153 
+
    +
    +
      [=========================>--------------] 67.40% | train_error: 0.152 
+
    +
    +
      [==========================>-------------] 67.50% | train_error: 0.151 
+
    +
    +
      [==========================>-------------] 67.60% | train_error: 0.151 
+
    +
    +
      [==========================>-------------] 67.70% | train_error: 0.150 
+
    +
    +
      [==========================>-------------] 67.80% | train_error: 0.149 
+
    +
    +
      [==========================>-------------] 67.90% | train_error: 0.149 
+
    +
    +
      [==========================>-------------] 68.00% | train_error: 0.148 
+
    +
    +
      [==========================>-------------] 68.10% | train_error: 0.147 
+
    +
    +
      [==========================>-------------] 68.20% | train_error: 0.146 
+
    +
    +
      [==========================>-------------] 68.30% | train_error: 0.146 
+
    +
    +
      [==========================>-------------] 68.40% | train_error: 0.145 
+
    +
    +
      [==========================>-------------] 68.50% | train_error: 0.144 
+
    +
    +
      [==========================>-------------] 68.60% | train_error: 0.144 
+
    +
    +
      [==========================>-------------] 68.70% | train_error: 0.143 
+
    +
    +
      [==========================>-------------] 68.80% | train_error: 0.142 
+
    +
    +
      [==========================>-------------] 68.90% | train_error: 0.142 
+
    +
    +
      [==========================>-------------] 69.00% | train_error: 0.141 
+
    +
    +
      [==========================>-------------] 69.10% | train_error: 0.141 
+
    +
    +
      [==========================>-------------] 69.20% | train_error: 0.140 
+
    +
    +
      [==========================>-------------] 69.30% | train_error: 0.139 
+
    +
    +
      [==========================>-------------] 69.40% | train_error: 0.139 
+
    +
    +
      [==========================>-------------] 69.50% | train_error: 0.138 
+
    +
    +
      [==========================>-------------] 69.60% | train_error: 0.137 
+
    +
    +
      [==========================>-------------] 69.70% | train_error: 0.137 
+
    +
    +
      [==========================>-------------] 69.80% | train_error: 0.136 
+
    +
    +
      [==========================>-------------] 69.90% | train_error: 0.135 
+
    +
    +
      [===========================>------------] 70.00% | train_error: 0.135 
+
    +
    +
      [===========================>------------] 70.10% | train_error: 0.134 
+
    +
    +
      [===========================>------------] 70.20% | train_error: 0.134 
+
    +
    +
      [===========================>------------] 70.30% | train_error: 0.133 
+
    +
    +
      [===========================>------------] 70.40% | train_error: 0.132 
+
    +
    +
      [===========================>------------] 70.50% | train_error: 0.132 
+
    +
    +
      [===========================>------------] 70.60% | train_error: 0.131 
+
    +
    +
      [===========================>------------] 70.70% | train_error: 0.131 
+
    +
    +
      [===========================>------------] 70.80% | train_error: 0.130 
+
    +
    +
      [===========================>------------] 70.90% | train_error: 0.129 
+
    +
    +
      [===========================>------------] 71.00% | train_error: 0.129 
+
    +
    +
      [===========================>------------] 71.10% | train_error: 0.128 
+
    +
    +
      [===========================>------------] 71.20% | train_error: 0.128 
+
    +
    +
      [===========================>------------] 71.30% | train_error: 0.127 
+
    +
    +
      [===========================>------------] 71.40% | train_error: 0.126 
+
    +
    +
      [===========================>------------] 71.50% | train_error: 0.126 
+
    +
    +
      [===========================>------------] 71.60% | train_error: 0.125 
+
    +
    +
      [===========================>------------] 71.70% | train_error: 0.125 
+
    +
    +
      [===========================>------------] 71.80% | train_error: 0.124 
+
    +
    +
      [===========================>------------] 71.90% | train_error: 0.124 
+
    +
    +
      [===========================>------------] 72.00% | train_error: 0.123 
+
    +
    +
      [===========================>------------] 72.10% | train_error: 0.122 
+
    +
    +
      [===========================>------------] 72.20% | train_error: 0.122 
+
    +
    +
      [===========================>------------] 72.30% | train_error: 0.121 
+
    +
    +
      [===========================>------------] 72.40% | train_error: 0.121 
+
    +
    +
      [============================>-----------] 72.50% | train_error: 0.120 
+
    +
    +
      [============================>-----------] 72.60% | train_error: 0.120 
+
    +
    +
      [============================>-----------] 72.70% | train_error: 0.119 
+
    +
    +
      [============================>-----------] 72.80% | train_error: 0.119 
+
    +
    +
      [============================>-----------] 72.90% | train_error: 0.118 
+
    +
    +
      [============================>-----------] 73.00% | train_error: 0.117 
+
    +
    +
      [============================>-----------] 73.10% | train_error: 0.117 
+
    +
    +
      [============================>-----------] 73.20% | train_error: 0.116 
+
    +
    +
      [============================>-----------] 73.30% | train_error: 0.116 
+
    +
    +
      [============================>-----------] 73.40% | train_error: 0.115 
+
    +
    +
      [============================>-----------] 73.50% | train_error: 0.115 
+
    +
    +
      [============================>-----------] 73.60% | train_error: 0.114 
+
    +
    +
      [============================>-----------] 73.70% | train_error: 0.114 
+
    +
    +
      [============================>-----------] 73.80% | train_error: 0.113 
+
    +
    +
      [============================>-----------] 73.90% | train_error: 0.113 
+
    +
    +
      [============================>-----------] 74.00% | train_error: 0.112 
+
    +
    +
      [============================>-----------] 74.10% | train_error: 0.112 
+
    +
    +
      [============================>-----------] 74.20% | train_error: 0.111 
+
    +
    +
      [============================>-----------] 74.30% | train_error: 0.111 
+
    +
    +
      [============================>-----------] 74.40% | train_error: 0.110 
+
    +
    +
      [============================>-----------] 74.50% | train_error: 0.110 
+
    +
    +
      [============================>-----------] 74.60% | train_error: 0.109 
+
    +
    +
      [============================>-----------] 74.70% | train_error: 0.109 
+
    +
    +
      [============================>-----------] 74.80% | train_error: 0.108 
+
    +
    +
      [============================>-----------] 74.90% | train_error: 0.108 
+
    +
    +
      [=============================>----------] 75.00% | train_error: 0.107 
+
    +
    +
      [=============================>----------] 75.10% | train_error: 0.107 
+
    +
    +
      [=============================>----------] 75.20% | train_error: 0.106 
+
    +
    +
      [=============================>----------] 75.30% | train_error: 0.106 
+
    +
    +
      [=============================>----------] 75.40% | train_error: 0.105 
+
    +
    +
      [=============================>----------] 75.50% | train_error: 0.105 
+
    +
    +
      [=============================>----------] 75.60% | train_error: 0.104 
+
    +
    +
      [=============================>----------] 75.70% | train_error: 0.104 
+
    +
    +
      [=============================>----------] 75.80% | train_error: 0.103 
+
    +
    +
      [=============================>----------] 75.90% | train_error: 0.103 
+
    +
    +
      [=============================>----------] 76.00% | train_error: 0.102 
+
    +
    +
      [=============================>----------] 76.10% | train_error: 0.102 
+
    +
    +
      [=============================>----------] 76.20% | train_error: 0.101 
+
    +
    +
      [=============================>----------] 76.30% | train_error: 0.101 
+
    +
    +
      [=============================>----------] 76.40% | train_error: 0.101 
+
    +
    +
      [=============================>----------] 76.50% | train_error: 0.100 
+
    +
    +
      [=============================>----------] 76.60% | train_error: 0.0996 
+
    +
    +
      [=============================>----------] 76.70% | train_error: 0.0992 
+
    +
    +
      [=============================>----------] 76.80% | train_error: 0.0987 
+
    +
    +
      [=============================>----------] 76.90% | train_error: 0.0983 
+
    +
    +
      [=============================>----------] 77.00% | train_error: 0.0978 
+
    +
    +
      [=============================>----------] 77.10% | train_error: 0.0974 
+
    +
    +
      [=============================>----------] 77.20% | train_error: 0.0969 
+
    +
    +
      [=============================>----------] 77.30% | train_error: 0.0965 
+
    +
    +
      [=============================>----------] 77.40% | train_error: 0.0961 
+
    +
    +
      [==============================>---------] 77.50% | train_error: 0.0956 
+
    +
    +
      [==============================>---------] 77.60% | train_error: 0.0952 
+
    +
    +
      [==============================>---------] 77.70% | train_error: 0.0948 
+
    +
    +
      [==============================>---------] 77.80% | train_error: 0.0943 
+
    +
    +
      [==============================>---------] 77.90% | train_error: 0.0939 
+
    +
    +
      [==============================>---------] 78.00% | train_error: 0.0935 
+
    +
    +
      [==============================>---------] 78.10% | train_error: 0.0930 
+
    +
    +
      [==============================>---------] 78.20% | train_error: 0.0926 
+
    +
    +
      [==============================>---------] 78.30% | train_error: 0.0922 
+
    +
    +
      [==============================>---------] 78.40% | train_error: 0.0918 
+
    +
    +
      [==============================>---------] 78.50% | train_error: 0.0914 
+
    +
    +
      [==============================>---------] 78.60% | train_error: 0.0910 
+
    +
    +
      [==============================>---------] 78.70% | train_error: 0.0905 
+
    +
    +
      [==============================>---------] 78.80% | train_error: 0.0901 
+
    +
    +
      [==============================>---------] 78.90% | train_error: 0.0897 
+
    +
    +
      [==============================>---------] 79.00% | train_error: 0.0893 
+
    +
    +
      [==============================>---------] 79.10% | train_error: 0.0889 
+
    +
    +
      [==============================>---------] 79.20% | train_error: 0.0885 
+
    +
    +
      [==============================>---------] 79.30% | train_error: 0.0881 
+
    +
    +
      [==============================>---------] 79.40% | train_error: 0.0877 
+
    +
    +
      [==============================>---------] 79.50% | train_error: 0.0873 
+
    +
    +
      [==============================>---------] 79.60% | train_error: 0.0869 
+
    +
    +
      [==============================>---------] 79.70% | train_error: 0.0865 
+
    +
    +
      [==============================>---------] 79.80% | train_error: 0.0861 
+
    +
    +
      [==============================>---------] 79.90% | train_error: 0.0858 
+
    +
    +
      [===============================>--------] 80.00% | train_error: 0.0854 
+
    +
    +
      [===============================>--------] 80.10% | train_error: 0.0850 
+
    +
    +
      [===============================>--------] 80.20% | train_error: 0.0846 
+
    +
    +
      [===============================>--------] 80.30% | train_error: 0.0842 
+
    +
    +
      [===============================>--------] 80.40% | train_error: 0.0838 
+
    +
    +
      [===============================>--------] 80.50% | train_error: 0.0835 
+
    +
    +
      [===============================>--------] 80.60% | train_error: 0.0831 
+
    +
    +
      [===============================>--------] 80.70% | train_error: 0.0827 
+
    +
    +
      [===============================>--------] 80.80% | train_error: 0.0823 
+
    +
    +
      [===============================>--------] 80.90% | train_error: 0.0820 
+
    +
    +
      [===============================>--------] 81.00% | train_error: 0.0816 
+
    +
    +
      [===============================>--------] 81.10% | train_error: 0.0812 
+
    +
    +
      [===============================>--------] 81.20% | train_error: 0.0809 
+
    +
    +
      [===============================>--------] 81.30% | train_error: 0.0805 
+
    +
    +
      [===============================>--------] 81.40% | train_error: 0.0801 
+
    +
    +
      [===============================>--------] 81.50% | train_error: 0.0798 
+
    +
    +
      [===============================>--------] 81.60% | train_error: 0.0794 
+
    +
    +
      [===============================>--------] 81.70% | train_error: 0.0791 
+
    +
    +
      [===============================>--------] 81.80% | train_error: 0.0787 
+
    +
    +
      [===============================>--------] 81.90% | train_error: 0.0784 
+
    +
    +
      [===============================>--------] 82.00% | train_error: 0.0780 
+
    +
    +
      [===============================>--------] 82.10% | train_error: 0.0776 
+
    +
    +
      [===============================>--------] 82.20% | train_error: 0.0773 
+
    +
    +
      [===============================>--------] 82.30% | train_error: 0.0770 
+
    +
    +
      [===============================>--------] 82.40% | train_error: 0.0766 
+
    +
    +
      [================================>-------] 82.50% | train_error: 0.0763 
+
    +
    +
      [================================>-------] 82.60% | train_error: 0.0759 
+
    +
    +
      [================================>-------] 82.70% | train_error: 0.0756 
+
    +
    +
      [================================>-------] 82.80% | train_error: 0.0752 
+
    +
    +
      [================================>-------] 82.90% | train_error: 0.0749 
+
    +
    +
      [================================>-------] 83.00% | train_error: 0.0746 
+
    +
    +
      [================================>-------] 83.10% | train_error: 0.0742 
+
    +
    +
      [================================>-------] 83.20% | train_error: 0.0739 
+
    +
    +
      [================================>-------] 83.30% | train_error: 0.0736 
+
    +
    +
      [================================>-------] 83.40% | train_error: 0.0732 
+
    +
    +
      [================================>-------] 83.50% | train_error: 0.0729 
+
    +
    +
      [================================>-------] 83.60% | train_error: 0.0726 
+
    +
    +
      [================================>-------] 83.70% | train_error: 0.0723 
+
    +
    +
      [================================>-------] 83.80% | train_error: 0.0719 
+
    +
    +
      [================================>-------] 83.90% | train_error: 0.0716 
+
    +
    +
      [================================>-------] 84.00% | train_error: 0.0713 
+
    +
    +
      [================================>-------] 84.10% | train_error: 0.0710 
+
    +
    +
      [================================>-------] 84.20% | train_error: 0.0707 
+
    +
    +
      [================================>-------] 84.30% | train_error: 0.0703 
+
    +
    +
      [================================>-------] 84.40% | train_error: 0.0700 
+
    +
    +
      [================================>-------] 84.50% | train_error: 0.0697 
+
    +
    +
      [================================>-------] 84.60% | train_error: 0.0694 
+
    +
    +
      [================================>-------] 84.70% | train_error: 0.0691 
+
    +
    +
      [================================>-------] 84.80% | train_error: 0.0688 
+
    +
    +
      [================================>-------] 84.90% | train_error: 0.0685 
+
    +
    +
      [=================================>------] 85.00% | train_error: 0.0682 
+
    +
    +
      [=================================>------] 85.10% | train_error: 0.0679 
+
    +
    +
      [=================================>------] 85.20% | train_error: 0.0676 
+
    +
    +
      [=================================>------] 85.30% | train_error: 0.0673 
+
    +
    +
      [=================================>------] 85.40% | train_error: 0.0670 
+
    +
    +
      [=================================>------] 85.50% | train_error: 0.0667 
+
    +
    +
      [=================================>------] 85.60% | train_error: 0.0664 
+
    +
    +
      [=================================>------] 85.70% | train_error: 0.0661 
+
    +
    +
      [=================================>------] 85.80% | train_error: 0.0658 
+
    +
    +
      [=================================>------] 85.90% | train_error: 0.0655 
+
    +
    +
      [=================================>------] 86.00% | train_error: 0.0652 
+
    +
    +
      [=================================>------] 86.10% | train_error: 0.0649 
+
    +
    +
      [=================================>------] 86.20% | train_error: 0.0646 
+
    +
    +
      [=================================>------] 86.30% | train_error: 0.0643 
+
    +
    +
      [=================================>------] 86.40% | train_error: 0.0641 
+
    +
    +
      [=================================>------] 86.50% | train_error: 0.0638 
+
    +
    +
      [=================================>------] 86.60% | train_error: 0.0635 
+
    +
    +
      [=================================>------] 86.70% | train_error: 0.0632 
+
    +
    +
      [=================================>------] 86.80% | train_error: 0.0629 
+
    +
    +
      [=================================>------] 86.90% | train_error: 0.0626 
+
    +
    +
      [=================================>------] 87.00% | train_error: 0.0624 
+
    +
    +
      [=================================>------] 87.10% | train_error: 0.0621 
+
    +
    +
      [=================================>------] 87.20% | train_error: 0.0618 
+
    +
    +
      [=================================>------] 87.30% | train_error: 0.0615 
+
    +
    +
      [=================================>------] 87.40% | train_error: 0.0613 
+
    +
    +
      [==================================>-----] 87.50% | train_error: 0.0610 
+
    +
    +
      [==================================>-----] 87.60% | train_error: 0.0607 
+
    +
    +
      [==================================>-----] 87.70% | train_error: 0.0605 
+
    +
    +
      [==================================>-----] 87.80% | train_error: 0.0602 
+
    +
    +
      [==================================>-----] 87.90% | train_error: 0.0599 
+
    +
    +
      [==================================>-----] 88.00% | train_error: 0.0597 
+
    +
    +
      [==================================>-----] 88.10% | train_error: 0.0594 
+
    +
    +
      [==================================>-----] 88.20% | train_error: 0.0591 
+
    +
    +
      [==================================>-----] 88.30% | train_error: 0.0589 
+
    +
    +
      [==================================>-----] 88.40% | train_error: 0.0586 
+
    +
    +
      [==================================>-----] 88.50% | train_error: 0.0584 
+
    +
    +
      [==================================>-----] 88.60% | train_error: 0.0581 
+
    +
    +
      [==================================>-----] 88.70% | train_error: 0.0578 
+
    +
    +
      [==================================>-----] 88.80% | train_error: 0.0576 
+
    +
    +
      [==================================>-----] 88.90% | train_error: 0.0573 
+
    +
    +
      [==================================>-----] 89.00% | train_error: 0.0571 
+
    +
    +
      [==================================>-----] 89.10% | train_error: 0.0568 
+
    +
    +
      [==================================>-----] 89.20% | train_error: 0.0566 
+
    +
    +
      [==================================>-----] 89.30% | train_error: 0.0563 
+
    +
    +
      [==================================>-----] 89.40% | train_error: 0.0561 
+
    +
    +
      [==================================>-----] 89.50% | train_error: 0.0558 
+
    +
    +
      [==================================>-----] 89.60% | train_error: 0.0556 
+
    +
    +
      [==================================>-----] 89.70% | train_error: 0.0553 
+
    +
    +
      [==================================>-----] 89.80% | train_error: 0.0551 
+
    +
    +
      [==================================>-----] 89.90% | train_error: 0.0549 
+
    +
    +
      [===================================>----] 90.00% | train_error: 0.0546 
+
    +
    +
      [===================================>----] 90.10% | train_error: 0.0544 
+
    +
    +
      [===================================>----] 90.20% | train_error: 0.0541 
+
    +
    +
      [===================================>----] 90.30% | train_error: 0.0539 
+
    +
    +
      [===================================>----] 90.40% | train_error: 0.0537 
+
    +
    +
      [===================================>----] 90.50% | train_error: 0.0534 
+
    +
    +
      [===================================>----] 90.60% | train_error: 0.0532 
+
    +
    +
      [===================================>----] 90.70% | train_error: 0.0530 
+
    +
    +
      [===================================>----] 90.80% | train_error: 0.0527 
+
    +
    +
      [===================================>----] 90.90% | train_error: 0.0525 
+
    +
    +
      [===================================>----] 91.00% | train_error: 0.0523 
+
    +
    +
      [===================================>----] 91.10% | train_error: 0.0520 
+
    +
    +
      [===================================>----] 91.20% | train_error: 0.0518 
+
    +
    +
      [===================================>----] 91.30% | train_error: 0.0516 
+
    +
    +
      [===================================>----] 91.40% | train_error: 0.0514 
+
    +
    +
      [===================================>----] 91.50% | train_error: 0.0511 
+
    +
    +
      [===================================>----] 91.60% | train_error: 0.0509 
+
    +
    +
      [===================================>----] 91.70% | train_error: 0.0507 
+
    +
    +
      [===================================>----] 91.80% | train_error: 0.0505 
+
    +
    +
      [===================================>----] 91.90% | train_error: 0.0503 
+
    +
    +
      [===================================>----] 92.00% | train_error: 0.0500 
+
    +
    +
      [===================================>----] 92.10% | train_error: 0.0498 
+
    +
    +
      [===================================>----] 92.20% | train_error: 0.0496 
+
    +
    +
      [===================================>----] 92.30% | train_error: 0.0494 
+
    +
    +
      [===================================>----] 92.40% | train_error: 0.0492 
+
    +
    +
      [====================================>---] 92.50% | train_error: 0.0490 
+
    +
    +
      [====================================>---] 92.60% | train_error: 0.0487 
+
    +
    +
      [====================================>---] 92.70% | train_error: 0.0485 
+
    +
    +
      [====================================>---] 92.80% | train_error: 0.0483 
+
    +
    +
      [====================================>---] 92.90% | train_error: 0.0481 
+
    +
    +
      [====================================>---] 93.00% | train_error: 0.0479 
+
    +
    +
      [====================================>---] 93.10% | train_error: 0.0477 
+
    +
    +
      [====================================>---] 93.20% | train_error: 0.0475 
+
    +
    +
      [====================================>---] 93.30% | train_error: 0.0473 
+
    +
    +
      [====================================>---] 93.40% | train_error: 0.0471 
+
    +
    +
      [====================================>---] 93.50% | train_error: 0.0469 
+
    +
    +
      [====================================>---] 93.60% | train_error: 0.0467 
+
    +
    +
      [====================================>---] 93.70% | train_error: 0.0465 
+
    +
    +
      [====================================>---] 93.80% | train_error: 0.0463 
+
    +
    +
      [====================================>---] 93.90% | train_error: 0.0461 
+
    +
    +
      [====================================>---] 94.00% | train_error: 0.0459 
+
    +
    +
      [====================================>---] 94.10% | train_error: 0.0457 
+
    +
    +
      [====================================>---] 94.20% | train_error: 0.0455 
+
    +
    +
      [====================================>---] 94.30% | train_error: 0.0453 
+
    +
    +
      [====================================>---] 94.40% | train_error: 0.0451 
+
    +
    +
      [====================================>---] 94.50% | train_error: 0.0449 
+
    +
    +
      [====================================>---] 94.60% | train_error: 0.0447 
+
    +
    +
      [====================================>---] 94.70% | train_error: 0.0445 
+
    +
    +
      [====================================>---] 94.80% | train_error: 0.0443 
+
    +
    +
      [====================================>---] 94.90% | train_error: 0.0441 
+
    +
    +
      [=====================================>--] 95.00% | train_error: 0.0439 
+
    +
    +
      [=====================================>--] 95.10% | train_error: 0.0437 
+
    +
    +
      [=====================================>--] 95.20% | train_error: 0.0435 
+
    +
    +
      [=====================================>--] 95.30% | train_error: 0.0434 
+
    +
    +
      [=====================================>--] 95.40% | train_error: 0.0432 
+
    +
    +
      [=====================================>--] 95.50% | train_error: 0.0430 
+
    +
    +
      [=====================================>--] 95.60% | train_error: 0.0428 
+
    +
    +
      [=====================================>--] 95.70% | train_error: 0.0426 
+
    +
    +
      [=====================================>--] 95.80% | train_error: 0.0424 
+
    +
    +
      [=====================================>--] 95.90% | train_error: 0.0423 
+
    +
    +
      [=====================================>--] 96.00% | train_error: 0.0421 
+
    +
    +
      [=====================================>--] 96.10% | train_error: 0.0419 
+
    +
    +
      [=====================================>--] 96.20% | train_error: 0.0417 
+
    +
    +
      [=====================================>--] 96.30% | train_error: 0.0415 
+
    +
    +
      [=====================================>--] 96.40% | train_error: 0.0414 
+
    +
    +
      [=====================================>--] 96.50% | train_error: 0.0412 
+
    +
    +
      [=====================================>--] 96.60% | train_error: 0.0410 
+
    +
    +
      [=====================================>--] 96.70% | train_error: 0.0408 
+
    +
    +
      [=====================================>--] 96.80% | train_error: 0.0407 
+
    +
    +
      [=====================================>--] 96.90% | train_error: 0.0405 
+
    +
    +
      [=====================================>--] 97.00% | train_error: 0.0403 
+
    +
    +
      [=====================================>--] 97.10% | train_error: 0.0401 
+
    +
    +
      [=====================================>--] 97.20% | train_error: 0.0400 
+
    +
    +
      [=====================================>--] 97.30% | train_error: 0.0398 
+
    +
    +
      [=====================================>--] 97.40% | train_error: 0.0396 
+
    +
    +
      [======================================>-] 97.50% | train_error: 0.0395 
+
    +
    +
      [======================================>-] 97.60% | train_error: 0.0393 
+
    +
    +
      [======================================>-] 97.70% | train_error: 0.0391 
+
    +
    +
      [======================================>-] 97.80% | train_error: 0.0389 
+
    +
    +
      [======================================>-] 97.90% | train_error: 0.0388 
+
    +
    +
      [======================================>-] 98.00% | train_error: 0.0386 
+
    +
    +
      [======================================>-] 98.10% | train_error: 0.0385 
+
    +
    +
      [======================================>-] 98.20% | train_error: 0.0383 
+
    +
    +
      [======================================>-] 98.30% | train_error: 0.0381 
+
    +
    +
      [======================================>-] 98.40% | train_error: 0.0380 
+
    +
    +
      [======================================>-] 98.50% | train_error: 0.0378 
+
    +
    +
      [======================================>-] 98.60% | train_error: 0.0376 
+
    +
    +
      [======================================>-] 98.70% | train_error: 0.0375 
+
    +
    +
      [======================================>-] 98.80% | train_error: 0.0373 
+
    +
    +
      [======================================>-] 98.90% | train_error: 0.0372 
+
    +
    +
      [======================================>-] 99.00% | train_error: 0.0370 
+
    +
    +
      [======================================>-] 99.10% | train_error: 0.0369 
+
    +
    +
      [======================================>-] 99.20% | train_error: 0.0367 
+
    +
    +
      [======================================>-] 99.30% | train_error: 0.0365 
+
    +
    +
      [======================================>-] 99.40% | train_error: 0.0364 
+
    +
    +
      [======================================>-] 99.50% | train_error: 0.0362 
+
    +
    +
      [======================================>-] 99.60% | train_error: 0.0361 
+
    +
    +
      [======================================>-] 99.70% | train_error: 0.0359 
+
    +
    +
      [======================================>-] 99.80% | train_error: 0.0358 
+
    +
    +
      [======================================>-] 99.90% | train_error: 0.0356 
+
    +
    +
                                                                              
+
    +
    +
      [=======================================>] 100.0% | train_error: 0.0356 
+
    +
    +
    +
    +

    We see that given more epochs to train on, the regressor reaches a lower MSE.

    +

    Let us then switch to a binary classification. We use a binary +classification dataset, and follow a similar setup to the regression +case.

    +
    +
    +
    from sklearn.datasets import load_breast_cancer
+from sklearn.preprocessing import MinMaxScaler
+
+wisconsin = load_breast_cancer()
+X = wisconsin.data
+target = wisconsin.target
+target = target.reshape(target.shape[0], 1)
+
+X_train, X_val, t_train, t_val = train_test_split(X, target)
+
+scaler = MinMaxScaler()
+scaler.fit(X_train)
+X_train = scaler.transform(X_train)
+X_val = scaler.transform(X_val)
+
    +
    +
    +
    +
    +
    +
    input_nodes = X_train.shape[1]
+output_nodes = 1
+
+logistic_regression = FFNN((input_nodes, output_nodes), output_func=sigmoid, cost_func=CostLogReg, seed=2023)
+
    +
    +
    +
    +

    We will now make use of our validation data by passing it into our fit function as a keyword argument

    +
    +
    +
    logistic_regression.reset_weights() # reset weights such that previous runs or reruns don't affect the weights
+
+scheduler = Adam(eta=1e-3, rho=0.9, rho2=0.999)
+scores = logistic_regression.fit(X_train, t_train, scheduler, epochs=1000, X_val=X_val, t_val=t_val)
+
    +
    +
    +
    +
    Adam: Eta=0.001, Lambda=0
+
    +
    +
    
+
    +
    +
      [----------------------------------------] 0.000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 0.1000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 0.2000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 0.3000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 0.4000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 0.5000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 0.6000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 0.7000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 0.8000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 0.9000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 1.000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 1.100% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 1.200% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 1.300% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 1.400% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 1.500% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 1.600% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 1.700% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 1.800% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 1.900% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 2.000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 2.100% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 2.200% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 2.300% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [----------------------------------------] 2.400% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 2.500% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 2.600% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 2.700% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 2.800% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 2.900% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 3.000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 3.100% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 3.200% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 3.300% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 3.400% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 3.500% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 3.600% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 3.700% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 3.800% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 3.900% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 4.000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 4.100% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 4.200% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 4.300% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 4.400% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 4.500% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 4.600% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 4.700% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 4.800% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [>---------------------------------------] 4.900% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 5.000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 5.100% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 5.200% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 5.300% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 5.400% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 5.500% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 5.600% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 5.700% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 5.800% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 5.900% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 6.000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 6.100% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 6.200% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 6.300% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 6.400% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 6.500% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 6.600% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 6.700% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 6.800% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 6.900% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 7.000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 7.100% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [=>--------------------------------------] 7.200% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [=>--------------------------------------] 7.300% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [=>--------------------------------------] 7.400% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 7.500% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 7.600% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 7.700% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 7.800% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 7.900% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 8.000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 8.100% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 8.200% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 8.300% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 8.400% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 8.500% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 8.600% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 8.700% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 8.800% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 8.900% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 9.000% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 9.100% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 9.200% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 9.300% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 9.400% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 9.500% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 9.600% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 9.700% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 9.800% | train_error: 12.9 | train_acc: 0.376 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [==>-------------------------------------] 9.900% | train_error: 13.0 | train_acc: 0.373 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 10.00% | train_error: 13.0 | train_acc: 0.373 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 10.10% | train_error: 13.1 | train_acc: 0.366 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 10.20% | train_error: 13.1 | train_acc: 0.366 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 10.30% | train_error: 13.1 | train_acc: 0.366 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 10.40% | train_error: 13.1 | train_acc: 0.366 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 10.50% | train_error: 13.0 | train_acc: 0.371 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 10.60% | train_error: 13.0 | train_acc: 0.371 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 10.70% | train_error: 13.0 | train_acc: 0.371 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 10.80% | train_error: 13.1 | train_acc: 0.369 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 10.90% | train_error: 13.1 | train_acc: 0.369 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 11.00% | train_error: 13.1 | train_acc: 0.369 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 11.10% | train_error: 13.1 | train_acc: 0.369 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 11.20% | train_error: 13.1 | train_acc: 0.369 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 11.30% | train_error: 13.1 | train_acc: 0.369 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 11.40% | train_error: 13.1 | train_acc: 0.369 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 11.50% | train_error: 13.1 | train_acc: 0.369 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 11.60% | train_error: 13.1 | train_acc: 0.369 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 11.70% | train_error: 13.1 | train_acc: 0.369 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 11.80% | train_error: 13.1 | train_acc: 0.369 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 11.90% | train_error: 13.1 | train_acc: 0.366 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 12.00% | train_error: 13.1 | train_acc: 0.366 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 12.10% | train_error: 13.1 | train_acc: 0.366 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 12.20% | train_error: 13.1 | train_acc: 0.366 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 12.30% | train_error: 13.1 | train_acc: 0.366 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [===>------------------------------------] 12.40% | train_error: 13.1 | train_acc: 0.366 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [====>-----------------------------------] 12.50% | train_error: 13.1 | train_acc: 0.366 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [====>-----------------------------------] 12.60% | train_error: 13.1 | train_acc: 0.366 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [====>-----------------------------------] 12.70% | train_error: 13.1 | train_acc: 0.366 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [====>-----------------------------------] 12.80% | train_error: 13.1 | train_acc: 0.366 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [====>-----------------------------------] 12.90% | train_error: 13.1 | train_acc: 0.366 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [====>-----------------------------------] 13.00% | train_error: 13.1 | train_acc: 0.366 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [====>-----------------------------------] 13.10% | train_error: 13.2 | train_acc: 0.362 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [====>-----------------------------------] 13.20% | train_error: 13.2 | train_acc: 0.362 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [====>-----------------------------------] 13.30% | train_error: 13.2 | train_acc: 0.362 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [====>-----------------------------------] 13.40% | train_error: 13.2 | train_acc: 0.362 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [====>-----------------------------------] 13.50% | train_error: 13.2 | train_acc: 0.362 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [====>-----------------------------------] 13.60% | train_error: 13.2 | train_acc: 0.362 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [====>-----------------------------------] 13.70% | train_error: 13.2 | train_acc: 0.362 | val_error: 12.8 | val_acc: 0.385 
+
    +
    +
      [====>-----------------------------------] 13.80% | train_error: 13.3 | train_acc: 0.359 | val_error: 12.9 | val_acc: 0.378 
+
    +
    +
      [====>-----------------------------------] 13.90% | train_error: 13.4 | train_acc: 0.354 | val_error: 12.9 | val_acc: 0.378 
+
    +
    +
      [====>-----------------------------------] 14.00% | train_error: 13.4 | train_acc: 0.354 | val_error: 13.0 | val_acc: 0.371 
+
    +
    +
      [====>-----------------------------------] 14.10% | train_error: 13.4 | train_acc: 0.354 | val_error: 13.0 | val_acc: 0.371 
+
    +
    +
      [====>-----------------------------------] 14.20% | train_error: 13.4 | train_acc: 0.354 | val_error: 13.0 | val_acc: 0.371 
+
    +
    +
      [====>-----------------------------------] 14.30% | train_error: 13.4 | train_acc: 0.354 | val_error: 13.0 | val_acc: 0.371 
+
    +
    +
      [====>-----------------------------------] 14.40% | train_error: 13.4 | train_acc: 0.354 | val_error: 13.0 | val_acc: 0.371 
+
    +
    +
      [====>-----------------------------------] 14.50% | train_error: 13.4 | train_acc: 0.354 | val_error: 13.0 | val_acc: 0.371 
+
    +
    +
      [====>-----------------------------------] 14.60% | train_error: 13.4 | train_acc: 0.354 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [====>-----------------------------------] 14.70% | train_error: 13.4 | train_acc: 0.354 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [====>-----------------------------------] 14.80% | train_error: 13.4 | train_acc: 0.352 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [====>-----------------------------------] 14.90% | train_error: 13.4 | train_acc: 0.352 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [=====>----------------------------------] 15.00% | train_error: 13.5 | train_acc: 0.347 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [=====>----------------------------------] 15.10% | train_error: 13.6 | train_acc: 0.345 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [=====>----------------------------------] 15.20% | train_error: 13.6 | train_acc: 0.345 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [=====>----------------------------------] 15.30% | train_error: 13.8 | train_acc: 0.336 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [=====>----------------------------------] 15.40% | train_error: 13.8 | train_acc: 0.336 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [=====>----------------------------------] 15.50% | train_error: 13.8 | train_acc: 0.336 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [=====>----------------------------------] 15.60% | train_error: 13.8 | train_acc: 0.333 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [=====>----------------------------------] 15.70% | train_error: 13.8 | train_acc: 0.333 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [=====>----------------------------------] 15.80% | train_error: 13.8 | train_acc: 0.333 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [=====>----------------------------------] 15.90% | train_error: 13.8 | train_acc: 0.333 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [=====>----------------------------------] 16.00% | train_error: 13.8 | train_acc: 0.333 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [=====>----------------------------------] 16.10% | train_error: 13.8 | train_acc: 0.333 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [=====>----------------------------------] 16.20% | train_error: 13.8 | train_acc: 0.333 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [=====>----------------------------------] 16.30% | train_error: 14.0 | train_acc: 0.326 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [=====>----------------------------------] 16.40% | train_error: 14.0 | train_acc: 0.326 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [=====>----------------------------------] 16.50% | train_error: 14.0 | train_acc: 0.326 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [=====>----------------------------------] 16.60% | train_error: 14.0 | train_acc: 0.326 | val_error: 13.3 | val_acc: 0.357 
+
    +
    +
      [=====>----------------------------------] 16.70% | train_error: 14.1 | train_acc: 0.322 | val_error: 13.5 | val_acc: 0.350 
+
    +
    +
      [=====>----------------------------------] 16.80% | train_error: 14.1 | train_acc: 0.322 | val_error: 13.5 | val_acc: 0.350 
+
    +
    +
      [=====>----------------------------------] 16.90% | train_error: 14.1 | train_acc: 0.319 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [=====>----------------------------------] 17.00% | train_error: 14.1 | train_acc: 0.319 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [=====>----------------------------------] 17.10% | train_error: 14.1 | train_acc: 0.319 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [=====>----------------------------------] 17.20% | train_error: 14.2 | train_acc: 0.317 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [=====>----------------------------------] 17.30% | train_error: 14.2 | train_acc: 0.317 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [=====>----------------------------------] 17.40% | train_error: 14.2 | train_acc: 0.317 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 17.50% | train_error: 14.2 | train_acc: 0.317 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 17.60% | train_error: 14.3 | train_acc: 0.312 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 17.70% | train_error: 14.4 | train_acc: 0.308 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 17.80% | train_error: 14.4 | train_acc: 0.308 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 17.90% | train_error: 14.4 | train_acc: 0.308 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 18.00% | train_error: 14.4 | train_acc: 0.308 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 18.10% | train_error: 14.4 | train_acc: 0.305 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 18.20% | train_error: 14.4 | train_acc: 0.305 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 18.30% | train_error: 14.4 | train_acc: 0.305 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 18.40% | train_error: 14.5 | train_acc: 0.298 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 18.50% | train_error: 14.5 | train_acc: 0.298 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 18.60% | train_error: 14.6 | train_acc: 0.296 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 18.70% | train_error: 14.6 | train_acc: 0.293 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 18.80% | train_error: 14.7 | train_acc: 0.291 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 18.90% | train_error: 14.7 | train_acc: 0.291 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 19.00% | train_error: 14.7 | train_acc: 0.291 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 19.10% | train_error: 14.7 | train_acc: 0.291 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [======>---------------------------------] 19.20% | train_error: 14.7 | train_acc: 0.291 | val_error: 13.5 | val_acc: 0.350 
+
    +
    +
      [======>---------------------------------] 19.30% | train_error: 14.8 | train_acc: 0.286 | val_error: 13.5 | val_acc: 0.350 
+
    +
    +
      [======>---------------------------------] 19.40% | train_error: 14.8 | train_acc: 0.284 | val_error: 13.5 | val_acc: 0.350 
+
    +
    +
      [======>---------------------------------] 19.50% | train_error: 14.8 | train_acc: 0.284 | val_error: 13.5 | val_acc: 0.350 
+
    +
    +
      [======>---------------------------------] 19.60% | train_error: 14.9 | train_acc: 0.282 | val_error: 13.5 | val_acc: 0.350 
+
    +
    +
      [======>---------------------------------] 19.70% | train_error: 14.9 | train_acc: 0.282 | val_error: 13.5 | val_acc: 0.350 
+
    +
    +
      [======>---------------------------------] 19.80% | train_error: 14.9 | train_acc: 0.282 | val_error: 13.5 | val_acc: 0.350 
+
    +
    +
      [======>---------------------------------] 19.90% | train_error: 14.9 | train_acc: 0.282 | val_error: 13.5 | val_acc: 0.350 
+
    +
    +
      [=======>--------------------------------] 20.00% | train_error: 14.9 | train_acc: 0.279 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [=======>--------------------------------] 20.10% | train_error: 15.1 | train_acc: 0.272 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [=======>--------------------------------] 20.20% | train_error: 15.1 | train_acc: 0.272 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [=======>--------------------------------] 20.30% | train_error: 15.1 | train_acc: 0.272 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [=======>--------------------------------] 20.40% | train_error: 15.1 | train_acc: 0.270 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [=======>--------------------------------] 20.50% | train_error: 15.1 | train_acc: 0.270 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [=======>--------------------------------] 20.60% | train_error: 15.4 | train_acc: 0.258 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [=======>--------------------------------] 20.70% | train_error: 15.4 | train_acc: 0.258 | val_error: 13.8 | val_acc: 0.336 
+
    +
    +
      [=======>--------------------------------] 20.80% | train_error: 15.4 | train_acc: 0.258 | val_error: 13.8 | val_acc: 0.336 
+
    +
    +
      [=======>--------------------------------] 20.90% | train_error: 15.4 | train_acc: 0.258 | val_error: 13.8 | val_acc: 0.336 
+
    +
    +
      [=======>--------------------------------] 21.00% | train_error: 15.4 | train_acc: 0.258 | val_error: 13.8 | val_acc: 0.336 
+
    +
    +
      [=======>--------------------------------] 21.10% | train_error: 15.4 | train_acc: 0.258 | val_error: 13.8 | val_acc: 0.336 
+
    +
    +
      [=======>--------------------------------] 21.20% | train_error: 15.3 | train_acc: 0.261 | val_error: 13.9 | val_acc: 0.329 
+
    +
    +
      [=======>--------------------------------] 21.30% | train_error: 15.5 | train_acc: 0.254 | val_error: 13.9 | val_acc: 0.329 
+
    +
    +
      [=======>--------------------------------] 21.40% | train_error: 15.5 | train_acc: 0.254 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [=======>--------------------------------] 21.50% | train_error: 15.5 | train_acc: 0.254 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [=======>--------------------------------] 21.60% | train_error: 15.5 | train_acc: 0.254 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [=======>--------------------------------] 21.70% | train_error: 15.4 | train_acc: 0.256 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [=======>--------------------------------] 21.80% | train_error: 15.4 | train_acc: 0.256 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [=======>--------------------------------] 21.90% | train_error: 15.4 | train_acc: 0.256 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [=======>--------------------------------] 22.00% | train_error: 15.4 | train_acc: 0.256 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [=======>--------------------------------] 22.10% | train_error: 15.6 | train_acc: 0.249 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [=======>--------------------------------] 22.20% | train_error: 15.6 | train_acc: 0.249 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [=======>--------------------------------] 22.30% | train_error: 15.6 | train_acc: 0.249 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [=======>--------------------------------] 22.40% | train_error: 15.7 | train_acc: 0.244 | val_error: 14.3 | val_acc: 0.308 
+
    +
    +
      [========>-------------------------------] 22.50% | train_error: 15.7 | train_acc: 0.242 | val_error: 14.3 | val_acc: 0.308 
+
    +
    +
      [========>-------------------------------] 22.60% | train_error: 15.7 | train_acc: 0.242 | val_error: 14.3 | val_acc: 0.308 
+
    +
    +
      [========>-------------------------------] 22.70% | train_error: 15.7 | train_acc: 0.242 | val_error: 14.5 | val_acc: 0.301 
+
    +
    +
      [========>-------------------------------] 22.80% | train_error: 15.7 | train_acc: 0.242 | val_error: 14.5 | val_acc: 0.301 
+
    +
    +
      [========>-------------------------------] 22.90% | train_error: 15.8 | train_acc: 0.239 | val_error: 14.5 | val_acc: 0.301 
+
    +
    +
      [========>-------------------------------] 23.00% | train_error: 15.9 | train_acc: 0.235 | val_error: 14.5 | val_acc: 0.301 
+
    +
    +
      [========>-------------------------------] 23.10% | train_error: 15.9 | train_acc: 0.235 | val_error: 14.5 | val_acc: 0.301 
+
    +
    +
      [========>-------------------------------] 23.20% | train_error: 15.9 | train_acc: 0.232 | val_error: 14.6 | val_acc: 0.294 
+
    +
    +
      [========>-------------------------------] 23.30% | train_error: 15.9 | train_acc: 0.232 | val_error: 14.6 | val_acc: 0.294 
+
    +
    +
      [========>-------------------------------] 23.40% | train_error: 15.9 | train_acc: 0.232 | val_error: 14.6 | val_acc: 0.294 
+
    +
    +
      [========>-------------------------------] 23.50% | train_error: 15.9 | train_acc: 0.232 | val_error: 14.9 | val_acc: 0.280 
+
    +
    +
      [========>-------------------------------] 23.60% | train_error: 15.9 | train_acc: 0.232 | val_error: 14.8 | val_acc: 0.287 
+
    +
    +
      [========>-------------------------------] 23.70% | train_error: 15.9 | train_acc: 0.232 | val_error: 14.8 | val_acc: 0.287 
+
    +
    +
      [========>-------------------------------] 23.80% | train_error: 16.0 | train_acc: 0.230 | val_error: 14.8 | val_acc: 0.287 
+
    +
    +
      [========>-------------------------------] 23.90% | train_error: 16.0 | train_acc: 0.230 | val_error: 14.9 | val_acc: 0.280 
+
    +
    +
      [========>-------------------------------] 24.00% | train_error: 16.0 | train_acc: 0.230 | val_error: 14.9 | val_acc: 0.280 
+
    +
    +
      [========>-------------------------------] 24.10% | train_error: 16.0 | train_acc: 0.230 | val_error: 15.1 | val_acc: 0.273 
+
    +
    +
      [========>-------------------------------] 24.20% | train_error: 16.0 | train_acc: 0.230 | val_error: 15.1 | val_acc: 0.273 
+
    +
    +
      [========>-------------------------------] 24.30% | train_error: 16.0 | train_acc: 0.230 | val_error: 15.1 | val_acc: 0.273 
+
    +
    +
      [========>-------------------------------] 24.40% | train_error: 16.0 | train_acc: 0.228 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [========>-------------------------------] 24.50% | train_error: 16.0 | train_acc: 0.228 | val_error: 15.4 | val_acc: 0.259 
+
    +
    +
      [========>-------------------------------] 24.60% | train_error: 16.0 | train_acc: 0.228 | val_error: 15.4 | val_acc: 0.259 
+
    +
    +
      [========>-------------------------------] 24.70% | train_error: 16.0 | train_acc: 0.228 | val_error: 15.4 | val_acc: 0.259 
+
    +
    +
      [========>-------------------------------] 24.80% | train_error: 16.0 | train_acc: 0.228 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [========>-------------------------------] 24.90% | train_error: 16.0 | train_acc: 0.228 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=========>------------------------------] 25.00% | train_error: 16.0 | train_acc: 0.228 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=========>------------------------------] 25.10% | train_error: 16.0 | train_acc: 0.228 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=========>------------------------------] 25.20% | train_error: 16.0 | train_acc: 0.228 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=========>------------------------------] 25.30% | train_error: 16.0 | train_acc: 0.228 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=========>------------------------------] 25.40% | train_error: 16.0 | train_acc: 0.228 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [=========>------------------------------] 25.50% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [=========>------------------------------] 25.60% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [=========>------------------------------] 25.70% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [=========>------------------------------] 25.80% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [=========>------------------------------] 25.90% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [=========>------------------------------] 26.00% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [=========>------------------------------] 26.10% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [=========>------------------------------] 26.20% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [=========>------------------------------] 26.30% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [=========>------------------------------] 26.40% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [=========>------------------------------] 26.50% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [=========>------------------------------] 26.60% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [=========>------------------------------] 26.70% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [=========>------------------------------] 26.80% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [=========>------------------------------] 26.90% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [=========>------------------------------] 27.00% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [=========>------------------------------] 27.10% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [=========>------------------------------] 27.20% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [=========>------------------------------] 27.30% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [=========>------------------------------] 27.40% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 27.50% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 27.60% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 27.70% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 27.80% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 27.90% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 28.00% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 28.10% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 28.20% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 28.30% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 28.40% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 28.50% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 28.60% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 28.70% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 28.80% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 28.90% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 29.00% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 29.10% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 29.20% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 29.30% | train_error: 16.0 | train_acc: 0.230 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 29.40% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 29.50% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 29.60% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 29.70% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 29.80% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [==========>-----------------------------] 29.90% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 30.00% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 30.10% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 30.20% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 30.30% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 30.40% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 30.50% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 30.60% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 30.70% | train_error: 15.7 | train_acc: 0.242 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 30.80% | train_error: 15.7 | train_acc: 0.242 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 30.90% | train_error: 15.7 | train_acc: 0.242 | val_error: 15.8 | val_acc: 0.238 
+
    +
    +
      [===========>----------------------------] 31.00% | train_error: 15.7 | train_acc: 0.242 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 31.10% | train_error: 15.7 | train_acc: 0.242 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 31.20% | train_error: 15.7 | train_acc: 0.242 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 31.30% | train_error: 15.7 | train_acc: 0.242 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 31.40% | train_error: 15.7 | train_acc: 0.242 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 31.50% | train_error: 15.7 | train_acc: 0.242 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 31.60% | train_error: 15.7 | train_acc: 0.244 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 31.70% | train_error: 15.5 | train_acc: 0.251 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 31.80% | train_error: 15.5 | train_acc: 0.251 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 31.90% | train_error: 15.6 | train_acc: 0.246 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 32.00% | train_error: 15.6 | train_acc: 0.246 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 32.10% | train_error: 15.6 | train_acc: 0.246 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 32.20% | train_error: 15.6 | train_acc: 0.246 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 32.30% | train_error: 15.6 | train_acc: 0.246 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [===========>----------------------------] 32.40% | train_error: 15.6 | train_acc: 0.246 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [============>---------------------------] 32.50% | train_error: 15.6 | train_acc: 0.246 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [============>---------------------------] 32.60% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [============>---------------------------] 32.70% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [============>---------------------------] 32.80% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [============>---------------------------] 32.90% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [============>---------------------------] 33.00% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [============>---------------------------] 33.10% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [============>---------------------------] 33.20% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [============>---------------------------] 33.30% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [============>---------------------------] 33.40% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [============>---------------------------] 33.50% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [============>---------------------------] 33.60% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [============>---------------------------] 33.70% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [============>---------------------------] 33.80% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [============>---------------------------] 33.90% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.7 | val_acc: 0.245 
+
    +
    +
      [============>---------------------------] 34.00% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [============>---------------------------] 34.10% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [============>---------------------------] 34.20% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [============>---------------------------] 34.30% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [============>---------------------------] 34.40% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [============>---------------------------] 34.50% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [============>---------------------------] 34.60% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [============>---------------------------] 34.70% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [============>---------------------------] 34.80% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [============>---------------------------] 34.90% | train_error: 15.9 | train_acc: 0.235 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=============>--------------------------] 35.00% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=============>--------------------------] 35.10% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=============>--------------------------] 35.20% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=============>--------------------------] 35.30% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=============>--------------------------] 35.40% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=============>--------------------------] 35.50% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=============>--------------------------] 35.60% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=============>--------------------------] 35.70% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=============>--------------------------] 35.80% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=============>--------------------------] 35.90% | train_error: 15.8 | train_acc: 0.237 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=============>--------------------------] 36.00% | train_error: 15.7 | train_acc: 0.244 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=============>--------------------------] 36.10% | train_error: 15.7 | train_acc: 0.244 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=============>--------------------------] 36.20% | train_error: 15.7 | train_acc: 0.244 | val_error: 15.5 | val_acc: 0.252 
+
    +
    +
      [=============>--------------------------] 36.30% | train_error: 15.7 | train_acc: 0.244 | val_error: 15.4 | val_acc: 0.259 
+
    +
    +
      [=============>--------------------------] 36.40% | train_error: 15.7 | train_acc: 0.244 | val_error: 15.4 | val_acc: 0.259 
+
    +
    +
      [=============>--------------------------] 36.50% | train_error: 15.7 | train_acc: 0.244 | val_error: 15.4 | val_acc: 0.259 
+
    +
    +
      [=============>--------------------------] 36.60% | train_error: 15.7 | train_acc: 0.244 | val_error: 15.4 | val_acc: 0.259 
+
    +
    +
      [=============>--------------------------] 36.70% | train_error: 15.7 | train_acc: 0.244 | val_error: 15.4 | val_acc: 0.259 
+
    +
    +
      [=============>--------------------------] 36.80% | train_error: 15.7 | train_acc: 0.244 | val_error: 15.4 | val_acc: 0.259 
+
    +
    +
      [=============>--------------------------] 36.90% | train_error: 15.7 | train_acc: 0.244 | val_error: 15.4 | val_acc: 0.259 
+
    +
    +
      [=============>--------------------------] 37.00% | train_error: 15.7 | train_acc: 0.244 | val_error: 15.4 | val_acc: 0.259 
+
    +
    +
      [=============>--------------------------] 37.10% | train_error: 15.7 | train_acc: 0.244 | val_error: 15.4 | val_acc: 0.259 
+
    +
    +
      [=============>--------------------------] 37.20% | train_error: 15.7 | train_acc: 0.244 | val_error: 15.4 | val_acc: 0.259 
+
    +
    +
      [=============>--------------------------] 37.30% | train_error: 15.7 | train_acc: 0.244 | val_error: 15.4 | val_acc: 0.259 
+
    +
    +
      [=============>--------------------------] 37.40% | train_error: 15.6 | train_acc: 0.249 | val_error: 15.4 | val_acc: 0.259 
+
    +
    +
      [==============>-------------------------] 37.50% | train_error: 15.6 | train_acc: 0.249 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 37.60% | train_error: 15.6 | train_acc: 0.249 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 37.70% | train_error: 15.6 | train_acc: 0.249 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 37.80% | train_error: 15.6 | train_acc: 0.249 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 37.90% | train_error: 15.6 | train_acc: 0.249 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 38.00% | train_error: 15.6 | train_acc: 0.249 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 38.10% | train_error: 15.6 | train_acc: 0.249 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 38.20% | train_error: 15.6 | train_acc: 0.249 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 38.30% | train_error: 15.6 | train_acc: 0.249 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 38.40% | train_error: 15.6 | train_acc: 0.249 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 38.50% | train_error: 15.6 | train_acc: 0.249 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 38.60% | train_error: 15.4 | train_acc: 0.256 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 38.70% | train_error: 15.3 | train_acc: 0.261 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 38.80% | train_error: 15.3 | train_acc: 0.261 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 38.90% | train_error: 15.4 | train_acc: 0.258 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 39.00% | train_error: 15.4 | train_acc: 0.258 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 39.10% | train_error: 15.4 | train_acc: 0.258 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 39.20% | train_error: 15.4 | train_acc: 0.258 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 39.30% | train_error: 15.2 | train_acc: 0.268 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 39.40% | train_error: 15.2 | train_acc: 0.268 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 39.50% | train_error: 15.2 | train_acc: 0.268 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 39.60% | train_error: 15.2 | train_acc: 0.268 | val_error: 15.2 | val_acc: 0.266 
+
    +
    +
      [==============>-------------------------] 39.70% | train_error: 15.2 | train_acc: 0.268 | val_error: 15.1 | val_acc: 0.273 
+
    +
    +
      [==============>-------------------------] 39.80% | train_error: 15.2 | train_acc: 0.268 | val_error: 15.1 | val_acc: 0.273 
+
    +
    +
      [==============>-------------------------] 39.90% | train_error: 15.1 | train_acc: 0.272 | val_error: 15.1 | val_acc: 0.273 
+
    +
    +
      [===============>------------------------] 40.00% | train_error: 15.1 | train_acc: 0.272 | val_error: 15.1 | val_acc: 0.273 
+
    +
    +
      [===============>------------------------] 40.10% | train_error: 15.1 | train_acc: 0.272 | val_error: 15.1 | val_acc: 0.273 
+
    +
    +
      [===============>------------------------] 40.20% | train_error: 15.1 | train_acc: 0.272 | val_error: 15.1 | val_acc: 0.273 
+
    +
    +
      [===============>------------------------] 40.30% | train_error: 15.1 | train_acc: 0.272 | val_error: 15.1 | val_acc: 0.273 
+
    +
    +
      [===============>------------------------] 40.40% | train_error: 15.1 | train_acc: 0.272 | val_error: 15.1 | val_acc: 0.273 
+
    +
    +
      [===============>------------------------] 40.50% | train_error: 15.0 | train_acc: 0.277 | val_error: 15.1 | val_acc: 0.273 
+
    +
    +
      [===============>------------------------] 40.60% | train_error: 15.0 | train_acc: 0.277 | val_error: 15.1 | val_acc: 0.273 
+
    +
    +
      [===============>------------------------] 40.70% | train_error: 15.0 | train_acc: 0.277 | val_error: 15.1 | val_acc: 0.273 
+
    +
    +
      [===============>------------------------] 40.80% | train_error: 15.0 | train_acc: 0.277 | val_error: 15.1 | val_acc: 0.273 
+
    +
    +
      [===============>------------------------] 40.90% | train_error: 15.0 | train_acc: 0.277 | val_error: 14.9 | val_acc: 0.280 
+
    +
    +
      [===============>------------------------] 41.00% | train_error: 15.0 | train_acc: 0.277 | val_error: 14.9 | val_acc: 0.280 
+
    +
    +
      [===============>------------------------] 41.10% | train_error: 14.9 | train_acc: 0.279 | val_error: 14.9 | val_acc: 0.280 
+
    +
    +
      [===============>------------------------] 41.20% | train_error: 14.9 | train_acc: 0.279 | val_error: 14.6 | val_acc: 0.294 
+
    +
    +
      [===============>------------------------] 41.30% | train_error: 14.9 | train_acc: 0.279 | val_error: 14.5 | val_acc: 0.301 
+
    +
    +
      [===============>------------------------] 41.40% | train_error: 14.9 | train_acc: 0.279 | val_error: 14.5 | val_acc: 0.301 
+
    +
    +
      [===============>------------------------] 41.50% | train_error: 14.9 | train_acc: 0.279 | val_error: 14.3 | val_acc: 0.308 
+
    +
    +
      [===============>------------------------] 41.60% | train_error: 15.0 | train_acc: 0.277 | val_error: 14.3 | val_acc: 0.308 
+
    +
    +
      [===============>------------------------] 41.70% | train_error: 15.0 | train_acc: 0.277 | val_error: 14.3 | val_acc: 0.308 
+
    +
    +
      [===============>------------------------] 41.80% | train_error: 14.9 | train_acc: 0.279 | val_error: 14.3 | val_acc: 0.308 
+
    +
    +
      [===============>------------------------] 41.90% | train_error: 14.9 | train_acc: 0.279 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [===============>------------------------] 42.00% | train_error: 14.9 | train_acc: 0.279 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [===============>------------------------] 42.10% | train_error: 14.9 | train_acc: 0.279 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [===============>------------------------] 42.20% | train_error: 14.9 | train_acc: 0.279 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [===============>------------------------] 42.30% | train_error: 14.9 | train_acc: 0.279 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [===============>------------------------] 42.40% | train_error: 14.9 | train_acc: 0.279 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 42.50% | train_error: 14.9 | train_acc: 0.279 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 42.60% | train_error: 14.9 | train_acc: 0.279 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 42.70% | train_error: 14.9 | train_acc: 0.279 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 42.80% | train_error: 14.9 | train_acc: 0.279 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 42.90% | train_error: 14.9 | train_acc: 0.279 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 43.00% | train_error: 14.9 | train_acc: 0.282 | val_error: 14.1 | val_acc: 0.322 
+
    +
    +
      [================>-----------------------] 43.10% | train_error: 14.9 | train_acc: 0.282 | val_error: 14.1 | val_acc: 0.322 
+
    +
    +
      [================>-----------------------] 43.20% | train_error: 14.9 | train_acc: 0.282 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 43.30% | train_error: 14.9 | train_acc: 0.282 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 43.40% | train_error: 14.9 | train_acc: 0.282 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 43.50% | train_error: 14.9 | train_acc: 0.282 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 43.60% | train_error: 14.9 | train_acc: 0.282 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 43.70% | train_error: 14.9 | train_acc: 0.282 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 43.80% | train_error: 14.9 | train_acc: 0.282 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 43.90% | train_error: 14.9 | train_acc: 0.282 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 44.00% | train_error: 14.8 | train_acc: 0.284 | val_error: 14.1 | val_acc: 0.322 
+
    +
    +
      [================>-----------------------] 44.10% | train_error: 14.8 | train_acc: 0.284 | val_error: 14.1 | val_acc: 0.322 
+
    +
    +
      [================>-----------------------] 44.20% | train_error: 14.7 | train_acc: 0.289 | val_error: 14.1 | val_acc: 0.322 
+
    +
    +
      [================>-----------------------] 44.30% | train_error: 14.7 | train_acc: 0.289 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 44.40% | train_error: 14.7 | train_acc: 0.289 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 44.50% | train_error: 14.7 | train_acc: 0.289 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 44.60% | train_error: 14.7 | train_acc: 0.289 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 44.70% | train_error: 14.7 | train_acc: 0.289 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 44.80% | train_error: 14.7 | train_acc: 0.289 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [================>-----------------------] 44.90% | train_error: 14.7 | train_acc: 0.289 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [=================>----------------------] 45.00% | train_error: 14.7 | train_acc: 0.289 | val_error: 14.2 | val_acc: 0.315 
+
    +
    +
      [=================>----------------------] 45.10% | train_error: 14.7 | train_acc: 0.291 | val_error: 14.1 | val_acc: 0.322 
+
    +
    +
      [=================>----------------------] 45.20% | train_error: 14.6 | train_acc: 0.296 | val_error: 14.1 | val_acc: 0.322 
+
    +
    +
      [=================>----------------------] 45.30% | train_error: 14.6 | train_acc: 0.296 | val_error: 14.1 | val_acc: 0.322 
+
    +
    +
      [=================>----------------------] 45.40% | train_error: 14.5 | train_acc: 0.300 | val_error: 14.1 | val_acc: 0.322 
+
    +
    +
      [=================>----------------------] 45.50% | train_error: 14.5 | train_acc: 0.300 | val_error: 14.1 | val_acc: 0.322 
+
    +
    +
      [=================>----------------------] 45.60% | train_error: 14.4 | train_acc: 0.308 | val_error: 14.1 | val_acc: 0.322 
+
    +
    +
      [=================>----------------------] 45.70% | train_error: 14.4 | train_acc: 0.308 | val_error: 14.1 | val_acc: 0.322 
+
    +
    +
      [=================>----------------------] 45.80% | train_error: 14.4 | train_acc: 0.308 | val_error: 13.9 | val_acc: 0.329 
+
    +
    +
      [=================>----------------------] 45.90% | train_error: 14.4 | train_acc: 0.308 | val_error: 13.9 | val_acc: 0.329 
+
    +
    +
      [=================>----------------------] 46.00% | train_error: 14.3 | train_acc: 0.310 | val_error: 13.9 | val_acc: 0.329 
+
    +
    +
      [=================>----------------------] 46.10% | train_error: 14.3 | train_acc: 0.310 | val_error: 13.9 | val_acc: 0.329 
+
    +
    +
      [=================>----------------------] 46.20% | train_error: 14.3 | train_acc: 0.312 | val_error: 13.9 | val_acc: 0.329 
+
    +
    +
      [=================>----------------------] 46.30% | train_error: 14.2 | train_acc: 0.315 | val_error: 13.9 | val_acc: 0.329 
+
    +
    +
      [=================>----------------------] 46.40% | train_error: 14.2 | train_acc: 0.315 | val_error: 13.9 | val_acc: 0.329 
+
    +
    +
      [=================>----------------------] 46.50% | train_error: 14.2 | train_acc: 0.315 | val_error: 13.9 | val_acc: 0.329 
+
    +
    +
      [=================>----------------------] 46.60% | train_error: 14.2 | train_acc: 0.315 | val_error: 13.9 | val_acc: 0.329 
+
    +
    +
      [=================>----------------------] 46.70% | train_error: 14.2 | train_acc: 0.315 | val_error: 13.9 | val_acc: 0.329 
+
    +
    +
      [=================>----------------------] 46.80% | train_error: 14.1 | train_acc: 0.319 | val_error: 13.8 | val_acc: 0.336 
+
    +
    +
      [=================>----------------------] 46.90% | train_error: 14.1 | train_acc: 0.319 | val_error: 13.8 | val_acc: 0.336 
+
    +
    +
      [=================>----------------------] 47.00% | train_error: 14.1 | train_acc: 0.322 | val_error: 13.8 | val_acc: 0.336 
+
    +
    +
      [=================>----------------------] 47.10% | train_error: 14.1 | train_acc: 0.322 | val_error: 13.8 | val_acc: 0.336 
+
    +
    +
      [=================>----------------------] 47.20% | train_error: 14.1 | train_acc: 0.322 | val_error: 13.8 | val_acc: 0.336 
+
    +
    +
      [=================>----------------------] 47.30% | train_error: 14.0 | train_acc: 0.324 | val_error: 13.8 | val_acc: 0.336 
+
    +
    +
      [=================>----------------------] 47.40% | train_error: 14.0 | train_acc: 0.324 | val_error: 13.8 | val_acc: 0.336 
+
    +
    +
      [==================>---------------------] 47.50% | train_error: 14.0 | train_acc: 0.324 | val_error: 13.8 | val_acc: 0.336 
+
    +
    +
      [==================>---------------------] 47.60% | train_error: 14.0 | train_acc: 0.326 | val_error: 13.6 | val_acc: 0.343 
+
    +
    +
      [==================>---------------------] 47.70% | train_error: 13.9 | train_acc: 0.329 | val_error: 13.5 | val_acc: 0.350 
+
    +
    +
      [==================>---------------------] 47.80% | train_error: 13.9 | train_acc: 0.329 | val_error: 13.5 | val_acc: 0.350 
+
    +
    +
      [==================>---------------------] 47.90% | train_error: 13.7 | train_acc: 0.338 | val_error: 13.5 | val_acc: 0.350 
+
    +
    +
      [==================>---------------------] 48.00% | train_error: 13.6 | train_acc: 0.343 | val_error: 13.3 | val_acc: 0.357 
+
    +
    +
      [==================>---------------------] 48.10% | train_error: 13.6 | train_acc: 0.343 | val_error: 13.3 | val_acc: 0.357 
+
    +
    +
      [==================>---------------------] 48.20% | train_error: 13.6 | train_acc: 0.343 | val_error: 13.3 | val_acc: 0.357 
+
    +
    +
      [==================>---------------------] 48.30% | train_error: 13.6 | train_acc: 0.343 | val_error: 13.3 | val_acc: 0.357 
+
    +
    +
      [==================>---------------------] 48.40% | train_error: 13.6 | train_acc: 0.343 | val_error: 13.3 | val_acc: 0.357 
+
    +
    +
      [==================>---------------------] 48.50% | train_error: 13.5 | train_acc: 0.347 | val_error: 13.3 | val_acc: 0.357 
+
    +
    +
      [==================>---------------------] 48.60% | train_error: 13.5 | train_acc: 0.347 | val_error: 13.3 | val_acc: 0.357 
+
    +
    +
      [==================>---------------------] 48.70% | train_error: 13.5 | train_acc: 0.347 | val_error: 13.3 | val_acc: 0.357 
+
    +
    +
      [==================>---------------------] 48.80% | train_error: 13.5 | train_acc: 0.347 | val_error: 13.2 | val_acc: 0.364 
+
    +
    +
      [==================>---------------------] 48.90% | train_error: 13.4 | train_acc: 0.352 | val_error: 12.9 | val_acc: 0.378 
+
    +
    +
      [==================>---------------------] 49.00% | train_error: 13.4 | train_acc: 0.352 | val_error: 12.9 | val_acc: 0.378 
+
    +
    +
      [==================>---------------------] 49.10% | train_error: 13.4 | train_acc: 0.352 | val_error: 12.9 | val_acc: 0.378 
+
    +
    +
      [==================>---------------------] 49.20% | train_error: 13.4 | train_acc: 0.352 | val_error: 12.9 | val_acc: 0.378 
+
    +
    +
      [==================>---------------------] 49.30% | train_error: 13.2 | train_acc: 0.362 | val_error: 12.9 | val_acc: 0.378 
+
    +
    +
      [==================>---------------------] 49.40% | train_error: 13.2 | train_acc: 0.362 | val_error: 12.9 | val_acc: 0.378 
+
    +
    +
      [==================>---------------------] 49.50% | train_error: 13.2 | train_acc: 0.362 | val_error: 12.9 | val_acc: 0.378 
+
    +
    +
      [==================>---------------------] 49.60% | train_error: 13.2 | train_acc: 0.362 | val_error: 12.9 | val_acc: 0.378 
+
    +
    +
      [==================>---------------------] 49.70% | train_error: 13.2 | train_acc: 0.362 | val_error: 12.9 | val_acc: 0.378 
+
    +
    +
      [==================>---------------------] 49.80% | train_error: 13.2 | train_acc: 0.362 | val_error: 12.9 | val_acc: 0.378 
+
    +
    +
      [==================>---------------------] 49.90% | train_error: 13.2 | train_acc: 0.362 | val_error: 12.9 | val_acc: 0.378 
+
    +
    +
      [===================>--------------------] 50.00% | train_error: 13.2 | train_acc: 0.364 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [===================>--------------------] 50.10% | train_error: 13.2 | train_acc: 0.364 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [===================>--------------------] 50.20% | train_error: 13.0 | train_acc: 0.373 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [===================>--------------------] 50.30% | train_error: 13.0 | train_acc: 0.373 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [===================>--------------------] 50.40% | train_error: 13.0 | train_acc: 0.371 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [===================>--------------------] 50.50% | train_error: 13.0 | train_acc: 0.371 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [===================>--------------------] 50.60% | train_error: 13.0 | train_acc: 0.371 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [===================>--------------------] 50.70% | train_error: 12.8 | train_acc: 0.380 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [===================>--------------------] 50.80% | train_error: 12.8 | train_acc: 0.383 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [===================>--------------------] 50.90% | train_error: 12.8 | train_acc: 0.383 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [===================>--------------------] 51.00% | train_error: 12.8 | train_acc: 0.383 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [===================>--------------------] 51.10% | train_error: 12.7 | train_acc: 0.385 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [===================>--------------------] 51.20% | train_error: 12.7 | train_acc: 0.385 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [===================>--------------------] 51.30% | train_error: 12.7 | train_acc: 0.385 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [===================>--------------------] 51.40% | train_error: 12.7 | train_acc: 0.385 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [===================>--------------------] 51.50% | train_error: 12.6 | train_acc: 0.390 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [===================>--------------------] 51.60% | train_error: 12.6 | train_acc: 0.390 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [===================>--------------------] 51.70% | train_error: 12.6 | train_acc: 0.390 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [===================>--------------------] 51.80% | train_error: 12.8 | train_acc: 0.380 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [===================>--------------------] 51.90% | train_error: 12.8 | train_acc: 0.380 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [===================>--------------------] 52.00% | train_error: 12.8 | train_acc: 0.380 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [===================>--------------------] 52.10% | train_error: 12.8 | train_acc: 0.380 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [===================>--------------------] 52.20% | train_error: 12.8 | train_acc: 0.383 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [===================>--------------------] 52.30% | train_error: 12.8 | train_acc: 0.383 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [===================>--------------------] 52.40% | train_error: 12.8 | train_acc: 0.383 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [====================>-------------------] 52.50% | train_error: 12.8 | train_acc: 0.383 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [====================>-------------------] 52.60% | train_error: 12.6 | train_acc: 0.390 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [====================>-------------------] 52.70% | train_error: 12.6 | train_acc: 0.390 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [====================>-------------------] 52.80% | train_error: 12.6 | train_acc: 0.390 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [====================>-------------------] 52.90% | train_error: 12.6 | train_acc: 0.390 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [====================>-------------------] 53.00% | train_error: 12.6 | train_acc: 0.390 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [====================>-------------------] 53.10% | train_error: 12.6 | train_acc: 0.390 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [====================>-------------------] 53.20% | train_error: 12.6 | train_acc: 0.392 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [====================>-------------------] 53.30% | train_error: 12.6 | train_acc: 0.392 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [====================>-------------------] 53.40% | train_error: 12.6 | train_acc: 0.394 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [====================>-------------------] 53.50% | train_error: 12.6 | train_acc: 0.394 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [====================>-------------------] 53.60% | train_error: 12.6 | train_acc: 0.394 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [====================>-------------------] 53.70% | train_error: 12.6 | train_acc: 0.394 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [====================>-------------------] 53.80% | train_error: 12.6 | train_acc: 0.394 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [====================>-------------------] 53.90% | train_error: 12.6 | train_acc: 0.394 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [====================>-------------------] 54.00% | train_error: 12.6 | train_acc: 0.394 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [====================>-------------------] 54.10% | train_error: 12.5 | train_acc: 0.397 | val_error: 12.3 | val_acc: 0.406 
+
    +
    +
      [====================>-------------------] 54.20% | train_error: 12.5 | train_acc: 0.397 | val_error: 12.3 | val_acc: 0.406 
+
    +
    +
      [====================>-------------------] 54.30% | train_error: 12.5 | train_acc: 0.397 | val_error: 12.3 | val_acc: 0.406 
+
    +
    +
      [====================>-------------------] 54.40% | train_error: 12.5 | train_acc: 0.397 | val_error: 12.3 | val_acc: 0.406 
+
    +
    +
      [====================>-------------------] 54.50% | train_error: 12.5 | train_acc: 0.397 | val_error: 12.3 | val_acc: 0.406 
+
    +
    +
      [====================>-------------------] 54.60% | train_error: 12.4 | train_acc: 0.401 | val_error: 12.3 | val_acc: 0.406 
+
    +
    +
      [====================>-------------------] 54.70% | train_error: 12.4 | train_acc: 0.404 | val_error: 12.3 | val_acc: 0.406 
+
    +
    +
      [====================>-------------------] 54.80% | train_error: 12.2 | train_acc: 0.411 | val_error: 12.3 | val_acc: 0.406 
+
    +
    +
      [====================>-------------------] 54.90% | train_error: 12.1 | train_acc: 0.415 | val_error: 12.2 | val_acc: 0.413 
+
    +
    +
      [=====================>------------------] 55.00% | train_error: 12.1 | train_acc: 0.415 | val_error: 12.0 | val_acc: 0.420 
+
    +
    +
      [=====================>------------------] 55.10% | train_error: 12.1 | train_acc: 0.415 | val_error: 12.0 | val_acc: 0.420 
+
    +
    +
      [=====================>------------------] 55.20% | train_error: 12.1 | train_acc: 0.415 | val_error: 12.0 | val_acc: 0.420 
+
    +
    +
      [=====================>------------------] 55.30% | train_error: 12.1 | train_acc: 0.418 | val_error: 12.2 | val_acc: 0.413 
+
    +
    +
      [=====================>------------------] 55.40% | train_error: 11.7 | train_acc: 0.437 | val_error: 12.0 | val_acc: 0.420 
+
    +
    +
      [=====================>------------------] 55.50% | train_error: 11.7 | train_acc: 0.434 | val_error: 12.0 | val_acc: 0.420 
+
    +
    +
      [=====================>------------------] 55.60% | train_error: 11.7 | train_acc: 0.434 | val_error: 12.0 | val_acc: 0.420 
+
    +
    +
      [=====================>------------------] 55.70% | train_error: 11.7 | train_acc: 0.437 | val_error: 12.0 | val_acc: 0.420 
+
    +
    +
      [=====================>------------------] 55.80% | train_error: 11.7 | train_acc: 0.437 | val_error: 11.9 | val_acc: 0.427 
+
    +
    +
      [=====================>------------------] 55.90% | train_error: 11.7 | train_acc: 0.437 | val_error: 11.9 | val_acc: 0.427 
+
    +
    +
      [=====================>------------------] 56.00% | train_error: 11.7 | train_acc: 0.437 | val_error: 11.9 | val_acc: 0.427 
+
    +
    +
      [=====================>------------------] 56.10% | train_error: 11.7 | train_acc: 0.437 | val_error: 11.7 | val_acc: 0.434 
+
    +
    +
      [=====================>------------------] 56.20% | train_error: 11.7 | train_acc: 0.437 | val_error: 11.7 | val_acc: 0.434 
+
    +
    +
      [=====================>------------------] 56.30% | train_error: 11.7 | train_acc: 0.437 | val_error: 11.7 | val_acc: 0.434 
+
    +
    +
      [=====================>------------------] 56.40% | train_error: 11.5 | train_acc: 0.444 | val_error: 11.7 | val_acc: 0.434 
+
    +
    +
      [=====================>------------------] 56.50% | train_error: 11.5 | train_acc: 0.444 | val_error: 11.7 | val_acc: 0.434 
+
    +
    +
      [=====================>------------------] 56.60% | train_error: 11.6 | train_acc: 0.441 | val_error: 11.7 | val_acc: 0.434 
+
    +
    +
      [=====================>------------------] 56.70% | train_error: 11.3 | train_acc: 0.453 | val_error: 11.7 | val_acc: 0.434 
+
    +
    +
      [=====================>------------------] 56.80% | train_error: 11.3 | train_acc: 0.453 | val_error: 11.6 | val_acc: 0.441 
+
    +
    +
      [=====================>------------------] 56.90% | train_error: 11.3 | train_acc: 0.453 | val_error: 11.6 | val_acc: 0.441 
+
    +
    +
      [=====================>------------------] 57.00% | train_error: 11.3 | train_acc: 0.453 | val_error: 11.6 | val_acc: 0.441 
+
    +
    +
      [=====================>------------------] 57.10% | train_error: 11.3 | train_acc: 0.453 | val_error: 11.6 | val_acc: 0.441 
+
    +
    +
      [=====================>------------------] 57.20% | train_error: 11.3 | train_acc: 0.453 | val_error: 11.6 | val_acc: 0.441 
+
    +
    +
      [=====================>------------------] 57.30% | train_error: 11.3 | train_acc: 0.453 | val_error: 11.6 | val_acc: 0.441 
+
    +
    +
      [=====================>------------------] 57.40% | train_error: 11.3 | train_acc: 0.453 | val_error: 11.6 | val_acc: 0.441 
+
    +
    +
      [======================>-----------------] 57.50% | train_error: 11.4 | train_acc: 0.451 | val_error: 11.6 | val_acc: 0.441 
+
    +
    +
      [======================>-----------------] 57.60% | train_error: 11.4 | train_acc: 0.451 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 57.70% | train_error: 11.4 | train_acc: 0.451 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 57.80% | train_error: 11.4 | train_acc: 0.451 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 57.90% | train_error: 11.3 | train_acc: 0.455 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 58.00% | train_error: 11.2 | train_acc: 0.460 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 58.10% | train_error: 10.9 | train_acc: 0.472 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 58.20% | train_error: 10.9 | train_acc: 0.472 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 58.30% | train_error: 10.9 | train_acc: 0.472 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 58.40% | train_error: 10.9 | train_acc: 0.472 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 58.50% | train_error: 10.9 | train_acc: 0.472 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 58.60% | train_error: 10.9 | train_acc: 0.472 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 58.70% | train_error: 10.8 | train_acc: 0.477 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 58.80% | train_error: 10.8 | train_acc: 0.477 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 58.90% | train_error: 10.8 | train_acc: 0.477 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 59.00% | train_error: 10.7 | train_acc: 0.484 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 59.10% | train_error: 10.7 | train_acc: 0.484 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 59.20% | train_error: 10.7 | train_acc: 0.486 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 59.30% | train_error: 10.7 | train_acc: 0.486 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [======================>-----------------] 59.40% | train_error: 10.7 | train_acc: 0.486 | val_error: 11.6 | val_acc: 0.441 
+
    +
    +
      [======================>-----------------] 59.50% | train_error: 10.7 | train_acc: 0.486 | val_error: 11.6 | val_acc: 0.441 
+
    +
    +
      [======================>-----------------] 59.60% | train_error: 10.7 | train_acc: 0.486 | val_error: 11.6 | val_acc: 0.441 
+
    +
    +
      [======================>-----------------] 59.70% | train_error: 10.6 | train_acc: 0.488 | val_error: 11.6 | val_acc: 0.441 
+
    +
    +
      [======================>-----------------] 59.80% | train_error: 10.6 | train_acc: 0.488 | val_error: 11.6 | val_acc: 0.441 
+
    +
    +
      [======================>-----------------] 59.90% | train_error: 10.6 | train_acc: 0.491 | val_error: 11.6 | val_acc: 0.441 
+
    +
    +
      [=======================>----------------] 60.00% | train_error: 10.5 | train_acc: 0.493 | val_error: 11.6 | val_acc: 0.441 
+
    +
    +
      [=======================>----------------] 60.10% | train_error: 10.5 | train_acc: 0.493 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [=======================>----------------] 60.20% | train_error: 10.5 | train_acc: 0.493 | val_error: 11.3 | val_acc: 0.455 
+
    +
    +
      [=======================>----------------] 60.30% | train_error: 10.5 | train_acc: 0.493 | val_error: 11.0 | val_acc: 0.469 
+
    +
    +
      [=======================>----------------] 60.40% | train_error: 10.5 | train_acc: 0.493 | val_error: 11.0 | val_acc: 0.469 
+
    +
    +
      [=======================>----------------] 60.50% | train_error: 10.5 | train_acc: 0.493 | val_error: 11.0 | val_acc: 0.469 
+
    +
    +
      [=======================>----------------] 60.60% | train_error: 10.3 | train_acc: 0.502 | val_error: 11.0 | val_acc: 0.469 
+
    +
    +
      [=======================>----------------] 60.70% | train_error: 10.3 | train_acc: 0.502 | val_error: 10.7 | val_acc: 0.483 
+
    +
    +
      [=======================>----------------] 60.80% | train_error: 10.3 | train_acc: 0.502 | val_error: 10.7 | val_acc: 0.483 
+
    +
    +
      [=======================>----------------] 60.90% | train_error: 10.3 | train_acc: 0.502 | val_error: 10.7 | val_acc: 0.483 
+
    +
    +
      [=======================>----------------] 61.00% | train_error: 10.2 | train_acc: 0.507 | val_error: 10.7 | val_acc: 0.483 
+
    +
    +
      [=======================>----------------] 61.10% | train_error: 10.2 | train_acc: 0.507 | val_error: 10.7 | val_acc: 0.483 
+
    +
    +
      [=======================>----------------] 61.20% | train_error: 10.2 | train_acc: 0.509 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [=======================>----------------] 61.30% | train_error: 10.2 | train_acc: 0.509 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [=======================>----------------] 61.40% | train_error: 10.2 | train_acc: 0.509 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [=======================>----------------] 61.50% | train_error: 10.2 | train_acc: 0.509 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [=======================>----------------] 61.60% | train_error: 10.1 | train_acc: 0.512 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [=======================>----------------] 61.70% | train_error: 10.1 | train_acc: 0.512 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [=======================>----------------] 61.80% | train_error: 10.1 | train_acc: 0.512 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [=======================>----------------] 61.90% | train_error: 10.1 | train_acc: 0.512 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [=======================>----------------] 62.00% | train_error: 10.0 | train_acc: 0.516 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [=======================>----------------] 62.10% | train_error: 9.97 | train_acc: 0.519 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [=======================>----------------] 62.20% | train_error: 9.97 | train_acc: 0.519 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [=======================>----------------] 62.30% | train_error: 9.88 | train_acc: 0.523 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [=======================>----------------] 62.40% | train_error: 9.88 | train_acc: 0.523 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [========================>---------------] 62.50% | train_error: 9.78 | train_acc: 0.528 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [========================>---------------] 62.60% | train_error: 9.68 | train_acc: 0.533 | val_error: 10.3 | val_acc: 0.503 
+
    +
    +
      [========================>---------------] 62.70% | train_error: 9.68 | train_acc: 0.533 | val_error: 10.00 | val_acc: 0.517 
+
    +
    +
      [========================>---------------] 62.80% | train_error: 9.53 | train_acc: 0.540 | val_error: 10.00 | val_acc: 0.517 
+
    +
    +
      [========================>---------------] 62.90% | train_error: 9.44 | train_acc: 0.545 | val_error: 10.00 | val_acc: 0.517 
+
    +
    +
      [========================>---------------] 63.00% | train_error: 9.39 | train_acc: 0.547 | val_error: 10.00 | val_acc: 0.517 
+
    +
    +
      [========================>---------------] 63.10% | train_error: 9.29 | train_acc: 0.552 | val_error: 10.00 | val_acc: 0.517 
+
    +
    +
      [========================>---------------] 63.20% | train_error: 9.29 | train_acc: 0.552 | val_error: 10.00 | val_acc: 0.517 
+
    +
    +
      [========================>---------------] 63.30% | train_error: 9.24 | train_acc: 0.554 | val_error: 9.85 | val_acc: 0.524 
+
    +
    +
      [========================>---------------] 63.40% | train_error: 9.24 | train_acc: 0.554 | val_error: 9.85 | val_acc: 0.524 
+
    +
    +
      [========================>---------------] 63.50% | train_error: 8.85 | train_acc: 0.573 | val_error: 9.85 | val_acc: 0.524 
+
    +
    +
      [========================>---------------] 63.60% | train_error: 8.85 | train_acc: 0.573 | val_error: 9.85 | val_acc: 0.524 
+
    +
    +
      [========================>---------------] 63.70% | train_error: 8.76 | train_acc: 0.577 | val_error: 9.56 | val_acc: 0.538 
+
    +
    +
      [========================>---------------] 63.80% | train_error: 8.76 | train_acc: 0.577 | val_error: 9.56 | val_acc: 0.538 
+
    +
    +
      [========================>---------------] 63.90% | train_error: 8.76 | train_acc: 0.577 | val_error: 9.56 | val_acc: 0.538 
+
    +
    +
      [========================>---------------] 64.00% | train_error: 8.76 | train_acc: 0.577 | val_error: 9.56 | val_acc: 0.538 
+
    +
    +
      [========================>---------------] 64.10% | train_error: 8.71 | train_acc: 0.580 | val_error: 9.56 | val_acc: 0.538 
+
    +
    +
      [========================>---------------] 64.20% | train_error: 8.71 | train_acc: 0.580 | val_error: 9.42 | val_acc: 0.545 
+
    +
    +
      [========================>---------------] 64.30% | train_error: 8.71 | train_acc: 0.580 | val_error: 9.42 | val_acc: 0.545 
+
    +
    +
      [========================>---------------] 64.40% | train_error: 8.56 | train_acc: 0.587 | val_error: 9.42 | val_acc: 0.545 
+
    +
    +
      [========================>---------------] 64.50% | train_error: 8.56 | train_acc: 0.587 | val_error: 9.42 | val_acc: 0.545 
+
    +
    +
      [========================>---------------] 64.60% | train_error: 8.51 | train_acc: 0.589 | val_error: 9.42 | val_acc: 0.545 
+
    +
    +
      [========================>---------------] 64.70% | train_error: 8.46 | train_acc: 0.592 | val_error: 9.42 | val_acc: 0.545 
+
    +
    +
      [========================>---------------] 64.80% | train_error: 8.32 | train_acc: 0.599 | val_error: 9.42 | val_acc: 0.545 
+
    +
    +
      [========================>---------------] 64.90% | train_error: 8.22 | train_acc: 0.603 | val_error: 9.42 | val_acc: 0.545 
+
    +
    +
      [=========================>--------------] 65.00% | train_error: 8.12 | train_acc: 0.608 | val_error: 9.42 | val_acc: 0.545 
+
    +
    +
      [=========================>--------------] 65.10% | train_error: 8.08 | train_acc: 0.610 | val_error: 9.27 | val_acc: 0.552 
+
    +
    +
      [=========================>--------------] 65.20% | train_error: 8.03 | train_acc: 0.613 | val_error: 9.27 | val_acc: 0.552 
+
    +
    +
      [=========================>--------------] 65.30% | train_error: 8.03 | train_acc: 0.613 | val_error: 9.27 | val_acc: 0.552 
+
    +
    +
      [=========================>--------------] 65.40% | train_error: 8.03 | train_acc: 0.613 | val_error: 9.13 | val_acc: 0.559 
+
    +
    +
      [=========================>--------------] 65.50% | train_error: 8.03 | train_acc: 0.613 | val_error: 9.13 | val_acc: 0.559 
+
    +
    +
      [=========================>--------------] 65.60% | train_error: 8.03 | train_acc: 0.613 | val_error: 8.98 | val_acc: 0.566 
+
    +
    +
      [=========================>--------------] 65.70% | train_error: 8.03 | train_acc: 0.613 | val_error: 8.98 | val_acc: 0.566 
+
    +
    +
      [=========================>--------------] 65.80% | train_error: 8.03 | train_acc: 0.613 | val_error: 8.84 | val_acc: 0.573 
+
    +
    +
      [=========================>--------------] 65.90% | train_error: 7.98 | train_acc: 0.615 | val_error: 8.70 | val_acc: 0.580 
+
    +
    +
      [=========================>--------------] 66.00% | train_error: 7.98 | train_acc: 0.615 | val_error: 8.55 | val_acc: 0.587 
+
    +
    +
      [=========================>--------------] 66.10% | train_error: 7.88 | train_acc: 0.620 | val_error: 8.55 | val_acc: 0.587 
+
    +
    +
      [=========================>--------------] 66.20% | train_error: 7.88 | train_acc: 0.620 | val_error: 8.55 | val_acc: 0.587 
+
    +
    +
      [=========================>--------------] 66.30% | train_error: 7.88 | train_acc: 0.620 | val_error: 8.70 | val_acc: 0.580 
+
    +
    +
      [=========================>--------------] 66.40% | train_error: 7.88 | train_acc: 0.620 | val_error: 8.70 | val_acc: 0.580 
+
    +
    +
      [=========================>--------------] 66.50% | train_error: 7.88 | train_acc: 0.620 | val_error: 8.70 | val_acc: 0.580 
+
    +
    +
      [=========================>--------------] 66.60% | train_error: 7.73 | train_acc: 0.627 | val_error: 8.70 | val_acc: 0.580 
+
    +
    +
      [=========================>--------------] 66.70% | train_error: 7.73 | train_acc: 0.627 | val_error: 8.70 | val_acc: 0.580 
+
    +
    +
      [=========================>--------------] 66.80% | train_error: 7.73 | train_acc: 0.627 | val_error: 8.70 | val_acc: 0.580 
+
    +
    +
      [=========================>--------------] 66.90% | train_error: 7.73 | train_acc: 0.627 | val_error: 8.70 | val_acc: 0.580 
+
    +
    +
      [=========================>--------------] 67.00% | train_error: 7.69 | train_acc: 0.629 | val_error: 8.55 | val_acc: 0.587 
+
    +
    +
      [=========================>--------------] 67.10% | train_error: 7.69 | train_acc: 0.629 | val_error: 8.55 | val_acc: 0.587 
+
    +
    +
      [=========================>--------------] 67.20% | train_error: 7.69 | train_acc: 0.629 | val_error: 8.41 | val_acc: 0.594 
+
    +
    +
      [=========================>--------------] 67.30% | train_error: 7.69 | train_acc: 0.629 | val_error: 8.41 | val_acc: 0.594 
+
    +
    +
      [=========================>--------------] 67.40% | train_error: 7.69 | train_acc: 0.629 | val_error: 8.26 | val_acc: 0.601 
+
    +
    +
      [==========================>-------------] 67.50% | train_error: 7.69 | train_acc: 0.629 | val_error: 8.12 | val_acc: 0.608 
+
    +
    +
      [==========================>-------------] 67.60% | train_error: 7.69 | train_acc: 0.629 | val_error: 8.12 | val_acc: 0.608 
+
    +
    +
      [==========================>-------------] 67.70% | train_error: 7.69 | train_acc: 0.629 | val_error: 7.97 | val_acc: 0.615 
+
    +
    +
      [==========================>-------------] 67.80% | train_error: 7.54 | train_acc: 0.636 | val_error: 7.97 | val_acc: 0.615 
+
    +
    +
      [==========================>-------------] 67.90% | train_error: 7.54 | train_acc: 0.636 | val_error: 7.97 | val_acc: 0.615 
+
    +
    +
      [==========================>-------------] 68.00% | train_error: 7.54 | train_acc: 0.636 | val_error: 7.97 | val_acc: 0.615 
+
    +
    +
      [==========================>-------------] 68.10% | train_error: 7.44 | train_acc: 0.641 | val_error: 7.97 | val_acc: 0.615 
+
    +
    +
      [==========================>-------------] 68.20% | train_error: 7.44 | train_acc: 0.641 | val_error: 7.97 | val_acc: 0.615 
+
    +
    +
      [==========================>-------------] 68.30% | train_error: 7.30 | train_acc: 0.648 | val_error: 7.97 | val_acc: 0.615 
+
    +
    +
      [==========================>-------------] 68.40% | train_error: 7.30 | train_acc: 0.648 | val_error: 7.97 | val_acc: 0.615 
+
    +
    +
      [==========================>-------------] 68.50% | train_error: 7.30 | train_acc: 0.648 | val_error: 7.83 | val_acc: 0.622 
+
    +
    +
      [==========================>-------------] 68.60% | train_error: 7.10 | train_acc: 0.657 | val_error: 7.83 | val_acc: 0.622 
+
    +
    +
      [==========================>-------------] 68.70% | train_error: 7.01 | train_acc: 0.662 | val_error: 7.83 | val_acc: 0.622 
+
    +
    +
      [==========================>-------------] 68.80% | train_error: 6.91 | train_acc: 0.667 | val_error: 7.97 | val_acc: 0.615 
+
    +
    +
      [==========================>-------------] 68.90% | train_error: 6.91 | train_acc: 0.667 | val_error: 7.97 | val_acc: 0.615 
+
    +
    +
      [==========================>-------------] 69.00% | train_error: 6.81 | train_acc: 0.671 | val_error: 7.83 | val_acc: 0.622 
+
    +
    +
      [==========================>-------------] 69.10% | train_error: 6.81 | train_acc: 0.671 | val_error: 7.68 | val_acc: 0.629 
+
    +
    +
      [==========================>-------------] 69.20% | train_error: 6.81 | train_acc: 0.671 | val_error: 7.68 | val_acc: 0.629 
+
    +
    +
      [==========================>-------------] 69.30% | train_error: 6.81 | train_acc: 0.671 | val_error: 7.68 | val_acc: 0.629 
+
    +
    +
      [==========================>-------------] 69.40% | train_error: 6.76 | train_acc: 0.674 | val_error: 7.68 | val_acc: 0.629 
+
    +
    +
      [==========================>-------------] 69.50% | train_error: 6.76 | train_acc: 0.674 | val_error: 7.68 | val_acc: 0.629 
+
    +
    +
      [==========================>-------------] 69.60% | train_error: 6.71 | train_acc: 0.676 | val_error: 7.68 | val_acc: 0.629 
+
    +
    +
      [==========================>-------------] 69.70% | train_error: 6.71 | train_acc: 0.676 | val_error: 7.68 | val_acc: 0.629 
+
    +
    +
      [==========================>-------------] 69.80% | train_error: 6.71 | train_acc: 0.676 | val_error: 7.68 | val_acc: 0.629 
+
    +
    +
      [==========================>-------------] 69.90% | train_error: 6.71 | train_acc: 0.676 | val_error: 7.68 | val_acc: 0.629 
+
    +
    +
      [===========================>------------] 70.00% | train_error: 6.91 | train_acc: 0.667 | val_error: 7.68 | val_acc: 0.629 
+
    +
    +
      [===========================>------------] 70.10% | train_error: 6.91 | train_acc: 0.667 | val_error: 7.68 | val_acc: 0.629 
+
    +
    +
      [===========================>------------] 70.20% | train_error: 6.86 | train_acc: 0.669 | val_error: 7.68 | val_acc: 0.629 
+
    +
    +
      [===========================>------------] 70.30% | train_error: 6.86 | train_acc: 0.669 | val_error: 7.68 | val_acc: 0.629 
+
    +
    +
      [===========================>------------] 70.40% | train_error: 6.86 | train_acc: 0.669 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [===========================>------------] 70.50% | train_error: 6.71 | train_acc: 0.676 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [===========================>------------] 70.60% | train_error: 6.71 | train_acc: 0.676 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [===========================>------------] 70.70% | train_error: 6.71 | train_acc: 0.676 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [===========================>------------] 70.80% | train_error: 6.71 | train_acc: 0.676 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [===========================>------------] 70.90% | train_error: 6.71 | train_acc: 0.676 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [===========================>------------] 71.00% | train_error: 6.71 | train_acc: 0.676 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [===========================>------------] 71.10% | train_error: 6.66 | train_acc: 0.678 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [===========================>------------] 71.20% | train_error: 6.57 | train_acc: 0.683 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [===========================>------------] 71.30% | train_error: 6.57 | train_acc: 0.683 | val_error: 7.39 | val_acc: 0.643 
+
    +
    +
      [===========================>------------] 71.40% | train_error: 6.57 | train_acc: 0.683 | val_error: 7.39 | val_acc: 0.643 
+
    +
    +
      [===========================>------------] 71.50% | train_error: 6.57 | train_acc: 0.683 | val_error: 7.39 | val_acc: 0.643 
+
    +
    +
      [===========================>------------] 71.60% | train_error: 6.57 | train_acc: 0.683 | val_error: 7.39 | val_acc: 0.643 
+
    +
    +
      [===========================>------------] 71.70% | train_error: 6.57 | train_acc: 0.683 | val_error: 7.25 | val_acc: 0.650 
+
    +
    +
      [===========================>------------] 71.80% | train_error: 6.52 | train_acc: 0.685 | val_error: 7.25 | val_acc: 0.650 
+
    +
    +
      [===========================>------------] 71.90% | train_error: 6.52 | train_acc: 0.685 | val_error: 7.25 | val_acc: 0.650 
+
    +
    +
      [===========================>------------] 72.00% | train_error: 6.52 | train_acc: 0.685 | val_error: 7.25 | val_acc: 0.650 
+
    +
    +
      [===========================>------------] 72.10% | train_error: 6.52 | train_acc: 0.685 | val_error: 7.25 | val_acc: 0.650 
+
    +
    +
      [===========================>------------] 72.20% | train_error: 6.47 | train_acc: 0.688 | val_error: 7.25 | val_acc: 0.650 
+
    +
    +
      [===========================>------------] 72.30% | train_error: 6.42 | train_acc: 0.690 | val_error: 7.10 | val_acc: 0.657 
+
    +
    +
      [===========================>------------] 72.40% | train_error: 6.42 | train_acc: 0.690 | val_error: 7.10 | val_acc: 0.657 
+
    +
    +
      [============================>-----------] 72.50% | train_error: 6.42 | train_acc: 0.690 | val_error: 7.10 | val_acc: 0.657 
+
    +
    +
      [============================>-----------] 72.60% | train_error: 6.42 | train_acc: 0.690 | val_error: 6.81 | val_acc: 0.671 
+
    +
    +
      [============================>-----------] 72.70% | train_error: 6.37 | train_acc: 0.692 | val_error: 6.81 | val_acc: 0.671 
+
    +
    +
      [============================>-----------] 72.80% | train_error: 6.32 | train_acc: 0.695 | val_error: 6.81 | val_acc: 0.671 
+
    +
    +
      [============================>-----------] 72.90% | train_error: 6.32 | train_acc: 0.695 | val_error: 6.81 | val_acc: 0.671 
+
    +
    +
      [============================>-----------] 73.00% | train_error: 6.32 | train_acc: 0.695 | val_error: 6.81 | val_acc: 0.671 
+
    +
    +
      [============================>-----------] 73.10% | train_error: 6.32 | train_acc: 0.695 | val_error: 6.81 | val_acc: 0.671 
+
    +
    +
      [============================>-----------] 73.20% | train_error: 6.32 | train_acc: 0.695 | val_error: 6.81 | val_acc: 0.671 
+
    +
    +
      [============================>-----------] 73.30% | train_error: 6.32 | train_acc: 0.695 | val_error: 6.81 | val_acc: 0.671 
+
    +
    +
      [============================>-----------] 73.40% | train_error: 6.32 | train_acc: 0.695 | val_error: 6.81 | val_acc: 0.671 
+
    +
    +
      [============================>-----------] 73.50% | train_error: 6.23 | train_acc: 0.700 | val_error: 6.81 | val_acc: 0.671 
+
    +
    +
      [============================>-----------] 73.60% | train_error: 6.23 | train_acc: 0.700 | val_error: 6.81 | val_acc: 0.671 
+
    +
    +
      [============================>-----------] 73.70% | train_error: 6.23 | train_acc: 0.700 | val_error: 6.67 | val_acc: 0.678 
+
    +
    +
      [============================>-----------] 73.80% | train_error: 6.23 | train_acc: 0.700 | val_error: 6.67 | val_acc: 0.678 
+
    +
    +
      [============================>-----------] 73.90% | train_error: 6.18 | train_acc: 0.702 | val_error: 6.67 | val_acc: 0.678 
+
    +
    +
      [============================>-----------] 74.00% | train_error: 5.98 | train_acc: 0.711 | val_error: 6.67 | val_acc: 0.678 
+
    +
    +
      [============================>-----------] 74.10% | train_error: 5.93 | train_acc: 0.714 | val_error: 6.67 | val_acc: 0.678 
+
    +
    +
      [============================>-----------] 74.20% | train_error: 5.84 | train_acc: 0.718 | val_error: 6.67 | val_acc: 0.678 
+
    +
    +
      [============================>-----------] 74.30% | train_error: 5.84 | train_acc: 0.718 | val_error: 6.67 | val_acc: 0.678 
+
    +
    +
      [============================>-----------] 74.40% | train_error: 5.84 | train_acc: 0.718 | val_error: 6.67 | val_acc: 0.678 
+
    +
    +
      [============================>-----------] 74.50% | train_error: 5.79 | train_acc: 0.721 | val_error: 6.67 | val_acc: 0.678 
+
    +
    +
      [============================>-----------] 74.60% | train_error: 5.64 | train_acc: 0.728 | val_error: 6.67 | val_acc: 0.678 
+
    +
    +
      [============================>-----------] 74.70% | train_error: 5.64 | train_acc: 0.728 | val_error: 6.67 | val_acc: 0.678 
+
    +
    +
      [============================>-----------] 74.80% | train_error: 5.59 | train_acc: 0.730 | val_error: 6.67 | val_acc: 0.678 
+
    +
    +
      [============================>-----------] 74.90% | train_error: 5.59 | train_acc: 0.730 | val_error: 6.38 | val_acc: 0.692 
+
    +
    +
      [=============================>----------] 75.00% | train_error: 5.59 | train_acc: 0.730 | val_error: 6.38 | val_acc: 0.692 
+
    +
    +
      [=============================>----------] 75.10% | train_error: 5.50 | train_acc: 0.735 | val_error: 6.23 | val_acc: 0.699 
+
    +
    +
      [=============================>----------] 75.20% | train_error: 5.30 | train_acc: 0.744 | val_error: 6.23 | val_acc: 0.699 
+
    +
    +
      [=============================>----------] 75.30% | train_error: 5.30 | train_acc: 0.744 | val_error: 6.23 | val_acc: 0.699 
+
    +
    +
      [=============================>----------] 75.40% | train_error: 5.30 | train_acc: 0.744 | val_error: 6.23 | val_acc: 0.699 
+
    +
    +
      [=============================>----------] 75.50% | train_error: 5.16 | train_acc: 0.751 | val_error: 6.23 | val_acc: 0.699 
+
    +
    +
      [=============================>----------] 75.60% | train_error: 5.11 | train_acc: 0.754 | val_error: 6.09 | val_acc: 0.706 
+
    +
    +
      [=============================>----------] 75.70% | train_error: 5.06 | train_acc: 0.756 | val_error: 5.94 | val_acc: 0.713 
+
    +
    +
      [=============================>----------] 75.80% | train_error: 5.06 | train_acc: 0.756 | val_error: 5.94 | val_acc: 0.713 
+
    +
    +
      [=============================>----------] 75.90% | train_error: 5.06 | train_acc: 0.756 | val_error: 5.94 | val_acc: 0.713 
+
    +
    +
      [=============================>----------] 76.00% | train_error: 5.06 | train_acc: 0.756 | val_error: 5.94 | val_acc: 0.713 
+
    +
    +
      [=============================>----------] 76.10% | train_error: 5.01 | train_acc: 0.758 | val_error: 5.94 | val_acc: 0.713 
+
    +
    +
      [=============================>----------] 76.20% | train_error: 5.01 | train_acc: 0.758 | val_error: 5.94 | val_acc: 0.713 
+
    +
    +
      [=============================>----------] 76.30% | train_error: 4.91 | train_acc: 0.763 | val_error: 5.80 | val_acc: 0.720 
+
    +
    +
      [=============================>----------] 76.40% | train_error: 4.91 | train_acc: 0.763 | val_error: 5.80 | val_acc: 0.720 
+
    +
    +
      [=============================>----------] 76.50% | train_error: 4.91 | train_acc: 0.763 | val_error: 5.80 | val_acc: 0.720 
+
    +
    +
      [=============================>----------] 76.60% | train_error: 4.91 | train_acc: 0.763 | val_error: 5.80 | val_acc: 0.720 
+
    +
    +
      [=============================>----------] 76.70% | train_error: 4.91 | train_acc: 0.763 | val_error: 5.80 | val_acc: 0.720 
+
    +
    +
      [=============================>----------] 76.80% | train_error: 4.91 | train_acc: 0.763 | val_error: 5.80 | val_acc: 0.720 
+
    +
    +
      [=============================>----------] 76.90% | train_error: 4.67 | train_acc: 0.775 | val_error: 5.80 | val_acc: 0.720 
+
    +
    +
      [=============================>----------] 77.00% | train_error: 4.67 | train_acc: 0.775 | val_error: 5.80 | val_acc: 0.720 
+
    +
    +
      [=============================>----------] 77.10% | train_error: 4.67 | train_acc: 0.775 | val_error: 5.80 | val_acc: 0.720 
+
    +
    +
      [=============================>----------] 77.20% | train_error: 4.67 | train_acc: 0.775 | val_error: 5.65 | val_acc: 0.727 
+
    +
    +
      [=============================>----------] 77.30% | train_error: 4.67 | train_acc: 0.775 | val_error: 5.65 | val_acc: 0.727 
+
    +
    +
      [=============================>----------] 77.40% | train_error: 4.67 | train_acc: 0.775 | val_error: 5.65 | val_acc: 0.727 
+
    +
    +
      [==============================>---------] 77.50% | train_error: 4.67 | train_acc: 0.775 | val_error: 5.65 | val_acc: 0.727 
+
    +
    +
      [==============================>---------] 77.60% | train_error: 4.67 | train_acc: 0.775 | val_error: 5.65 | val_acc: 0.727 
+
    +
    +
      [==============================>---------] 77.70% | train_error: 4.67 | train_acc: 0.775 | val_error: 5.36 | val_acc: 0.741 
+
    +
    +
      [==============================>---------] 77.80% | train_error: 4.67 | train_acc: 0.775 | val_error: 5.36 | val_acc: 0.741 
+
    +
    +
      [==============================>---------] 77.90% | train_error: 4.67 | train_acc: 0.775 | val_error: 5.36 | val_acc: 0.741 
+
    +
    +
      [==============================>---------] 78.00% | train_error: 4.67 | train_acc: 0.775 | val_error: 5.36 | val_acc: 0.741 
+
    +
    +
      [==============================>---------] 78.10% | train_error: 4.67 | train_acc: 0.775 | val_error: 5.36 | val_acc: 0.741 
+
    +
    +
      [==============================>---------] 78.20% | train_error: 4.62 | train_acc: 0.777 | val_error: 5.36 | val_acc: 0.741 
+
    +
    +
      [==============================>---------] 78.30% | train_error: 4.57 | train_acc: 0.779 | val_error: 5.22 | val_acc: 0.748 
+
    +
    +
      [==============================>---------] 78.40% | train_error: 4.57 | train_acc: 0.779 | val_error: 5.22 | val_acc: 0.748 
+
    +
    +
      [==============================>---------] 78.50% | train_error: 4.57 | train_acc: 0.779 | val_error: 5.22 | val_acc: 0.748 
+
    +
    +
      [==============================>---------] 78.60% | train_error: 4.57 | train_acc: 0.779 | val_error: 5.22 | val_acc: 0.748 
+
    +
    +
      [==============================>---------] 78.70% | train_error: 4.57 | train_acc: 0.779 | val_error: 5.22 | val_acc: 0.748 
+
    +
    +
      [==============================>---------] 78.80% | train_error: 4.57 | train_acc: 0.779 | val_error: 5.22 | val_acc: 0.748 
+
    +
    +
      [==============================>---------] 78.90% | train_error: 4.57 | train_acc: 0.779 | val_error: 5.07 | val_acc: 0.755 
+
    +
    +
      [==============================>---------] 79.00% | train_error: 4.52 | train_acc: 0.782 | val_error: 5.07 | val_acc: 0.755 
+
    +
    +
      [==============================>---------] 79.10% | train_error: 4.52 | train_acc: 0.782 | val_error: 4.93 | val_acc: 0.762 
+
    +
    +
      [==============================>---------] 79.20% | train_error: 4.43 | train_acc: 0.786 | val_error: 4.78 | val_acc: 0.769 
+
    +
    +
      [==============================>---------] 79.30% | train_error: 4.38 | train_acc: 0.789 | val_error: 4.78 | val_acc: 0.769 
+
    +
    +
      [==============================>---------] 79.40% | train_error: 4.13 | train_acc: 0.800 | val_error: 4.78 | val_acc: 0.769 
+
    +
    +
      [==============================>---------] 79.50% | train_error: 4.13 | train_acc: 0.800 | val_error: 4.78 | val_acc: 0.769 
+
    +
    +
      [==============================>---------] 79.60% | train_error: 4.13 | train_acc: 0.800 | val_error: 4.78 | val_acc: 0.769 
+
    +
    +
      [==============================>---------] 79.70% | train_error: 4.09 | train_acc: 0.803 | val_error: 4.78 | val_acc: 0.769 
+
    +
    +
      [==============================>---------] 79.80% | train_error: 4.09 | train_acc: 0.803 | val_error: 4.78 | val_acc: 0.769 
+
    +
    +
      [==============================>---------] 79.90% | train_error: 4.09 | train_acc: 0.803 | val_error: 4.64 | val_acc: 0.776 
+
    +
    +
      [===============================>--------] 80.00% | train_error: 4.09 | train_acc: 0.803 | val_error: 4.64 | val_acc: 0.776 
+
    +
    +
      [===============================>--------] 80.10% | train_error: 4.09 | train_acc: 0.803 | val_error: 4.49 | val_acc: 0.783 
+
    +
    +
      [===============================>--------] 80.20% | train_error: 4.04 | train_acc: 0.805 | val_error: 4.49 | val_acc: 0.783 
+
    +
    +
      [===============================>--------] 80.30% | train_error: 4.09 | train_acc: 0.803 | val_error: 4.49 | val_acc: 0.783 
+
    +
    +
      [===============================>--------] 80.40% | train_error: 4.09 | train_acc: 0.803 | val_error: 4.49 | val_acc: 0.783 
+
    +
    +
      [===============================>--------] 80.50% | train_error: 4.28 | train_acc: 0.793 | val_error: 4.35 | val_acc: 0.790 
+
    +
    +
      [===============================>--------] 80.60% | train_error: 4.28 | train_acc: 0.793 | val_error: 4.35 | val_acc: 0.790 
+
    +
    +
      [===============================>--------] 80.70% | train_error: 4.28 | train_acc: 0.793 | val_error: 4.35 | val_acc: 0.790 
+
    +
    +
      [===============================>--------] 80.80% | train_error: 4.28 | train_acc: 0.793 | val_error: 4.35 | val_acc: 0.790 
+
    +
    +
      [===============================>--------] 80.90% | train_error: 4.28 | train_acc: 0.793 | val_error: 4.35 | val_acc: 0.790 
+
    +
    +
      [===============================>--------] 81.00% | train_error: 4.28 | train_acc: 0.793 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [===============================>--------] 81.10% | train_error: 4.28 | train_acc: 0.793 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [===============================>--------] 81.20% | train_error: 4.28 | train_acc: 0.793 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [===============================>--------] 81.30% | train_error: 4.28 | train_acc: 0.793 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [===============================>--------] 81.40% | train_error: 4.28 | train_acc: 0.793 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [===============================>--------] 81.50% | train_error: 4.28 | train_acc: 0.793 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [===============================>--------] 81.60% | train_error: 4.28 | train_acc: 0.793 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [===============================>--------] 81.70% | train_error: 4.28 | train_acc: 0.793 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [===============================>--------] 81.80% | train_error: 4.28 | train_acc: 0.793 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [===============================>--------] 81.90% | train_error: 4.28 | train_acc: 0.793 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [===============================>--------] 82.00% | train_error: 4.28 | train_acc: 0.793 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [===============================>--------] 82.10% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [===============================>--------] 82.20% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [===============================>--------] 82.30% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [===============================>--------] 82.40% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [================================>-------] 82.50% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [================================>-------] 82.60% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [================================>-------] 82.70% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [================================>-------] 82.80% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [================================>-------] 82.90% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [================================>-------] 83.00% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [================================>-------] 83.10% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [================================>-------] 83.20% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [================================>-------] 83.30% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [================================>-------] 83.40% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [================================>-------] 83.50% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [================================>-------] 83.60% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [================================>-------] 83.70% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [================================>-------] 83.80% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [================================>-------] 83.90% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [================================>-------] 84.00% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [================================>-------] 84.10% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [================================>-------] 84.20% | train_error: 4.33 | train_acc: 0.791 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [================================>-------] 84.30% | train_error: 4.33 | train_acc: 0.791 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [================================>-------] 84.40% | train_error: 4.33 | train_acc: 0.791 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [================================>-------] 84.50% | train_error: 4.33 | train_acc: 0.791 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [================================>-------] 84.60% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [================================>-------] 84.70% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [================================>-------] 84.80% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [================================>-------] 84.90% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [=================================>------] 85.00% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [=================================>------] 85.10% | train_error: 4.23 | train_acc: 0.796 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [=================================>------] 85.20% | train_error: 4.18 | train_acc: 0.798 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [=================================>------] 85.30% | train_error: 4.18 | train_acc: 0.798 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [=================================>------] 85.40% | train_error: 4.18 | train_acc: 0.798 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [=================================>------] 85.50% | train_error: 4.18 | train_acc: 0.798 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [=================================>------] 85.60% | train_error: 4.13 | train_acc: 0.800 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [=================================>------] 85.70% | train_error: 4.13 | train_acc: 0.800 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [=================================>------] 85.80% | train_error: 4.09 | train_acc: 0.803 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [=================================>------] 85.90% | train_error: 4.09 | train_acc: 0.803 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [=================================>------] 86.00% | train_error: 4.09 | train_acc: 0.803 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [=================================>------] 86.10% | train_error: 4.09 | train_acc: 0.803 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [=================================>------] 86.20% | train_error: 4.09 | train_acc: 0.803 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [=================================>------] 86.30% | train_error: 4.09 | train_acc: 0.803 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [=================================>------] 86.40% | train_error: 4.09 | train_acc: 0.803 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [=================================>------] 86.50% | train_error: 4.09 | train_acc: 0.803 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [=================================>------] 86.60% | train_error: 4.04 | train_acc: 0.805 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [=================================>------] 86.70% | train_error: 4.04 | train_acc: 0.805 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [=================================>------] 86.80% | train_error: 4.04 | train_acc: 0.805 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [=================================>------] 86.90% | train_error: 3.99 | train_acc: 0.808 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [=================================>------] 87.00% | train_error: 3.89 | train_acc: 0.812 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [=================================>------] 87.10% | train_error: 3.89 | train_acc: 0.812 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [=================================>------] 87.20% | train_error: 3.89 | train_acc: 0.812 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [=================================>------] 87.30% | train_error: 3.89 | train_acc: 0.812 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [=================================>------] 87.40% | train_error: 3.79 | train_acc: 0.817 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [==================================>-----] 87.50% | train_error: 3.79 | train_acc: 0.817 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [==================================>-----] 87.60% | train_error: 3.89 | train_acc: 0.812 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [==================================>-----] 87.70% | train_error: 3.84 | train_acc: 0.815 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [==================================>-----] 87.80% | train_error: 3.84 | train_acc: 0.815 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [==================================>-----] 87.90% | train_error: 3.84 | train_acc: 0.815 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [==================================>-----] 88.00% | train_error: 3.84 | train_acc: 0.815 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [==================================>-----] 88.10% | train_error: 3.84 | train_acc: 0.815 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [==================================>-----] 88.20% | train_error: 3.84 | train_acc: 0.815 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [==================================>-----] 88.30% | train_error: 3.84 | train_acc: 0.815 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [==================================>-----] 88.40% | train_error: 3.84 | train_acc: 0.815 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [==================================>-----] 88.50% | train_error: 3.70 | train_acc: 0.822 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [==================================>-----] 88.60% | train_error: 3.70 | train_acc: 0.822 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [==================================>-----] 88.70% | train_error: 3.70 | train_acc: 0.822 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [==================================>-----] 88.80% | train_error: 3.70 | train_acc: 0.822 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [==================================>-----] 88.90% | train_error: 3.70 | train_acc: 0.822 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [==================================>-----] 89.00% | train_error: 3.70 | train_acc: 0.822 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [==================================>-----] 89.10% | train_error: 3.70 | train_acc: 0.822 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [==================================>-----] 89.20% | train_error: 3.70 | train_acc: 0.822 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [==================================>-----] 89.30% | train_error: 3.70 | train_acc: 0.822 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [==================================>-----] 89.40% | train_error: 3.70 | train_acc: 0.822 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [==================================>-----] 89.50% | train_error: 3.70 | train_acc: 0.822 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [==================================>-----] 89.60% | train_error: 3.70 | train_acc: 0.822 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [==================================>-----] 89.70% | train_error: 3.70 | train_acc: 0.822 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [==================================>-----] 89.80% | train_error: 3.65 | train_acc: 0.824 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [==================================>-----] 89.90% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [===================================>----] 90.00% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [===================================>----] 90.10% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [===================================>----] 90.20% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [===================================>----] 90.30% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [===================================>----] 90.40% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 90.50% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 90.60% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 90.70% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 90.80% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 90.90% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 91.00% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 91.10% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 91.20% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 91.30% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 91.40% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 91.50% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 91.60% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 91.70% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 91.80% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 91.90% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 92.00% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 92.10% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 92.20% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 92.30% | train_error: 3.60 | train_acc: 0.826 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [===================================>----] 92.40% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 92.50% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 92.60% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 92.70% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 92.80% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 92.90% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 93.00% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 93.10% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 93.20% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 93.30% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 93.40% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 93.50% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 93.60% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 93.70% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 93.80% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 93.90% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 94.00% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 94.10% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 94.20% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 94.30% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 94.40% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 94.50% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 94.60% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 94.70% | train_error: 3.65 | train_acc: 0.824 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 94.80% | train_error: 3.65 | train_acc: 0.824 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [====================================>---] 94.90% | train_error: 3.65 | train_acc: 0.824 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [=====================================>--] 95.00% | train_error: 3.65 | train_acc: 0.824 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [=====================================>--] 95.10% | train_error: 3.65 | train_acc: 0.824 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [=====================================>--] 95.20% | train_error: 3.65 | train_acc: 0.824 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [=====================================>--] 95.30% | train_error: 3.65 | train_acc: 0.824 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [=====================================>--] 95.40% | train_error: 3.65 | train_acc: 0.824 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [=====================================>--] 95.50% | train_error: 3.65 | train_acc: 0.824 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [=====================================>--] 95.60% | train_error: 3.65 | train_acc: 0.824 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 95.70% | train_error: 3.65 | train_acc: 0.824 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 95.80% | train_error: 3.65 | train_acc: 0.824 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 95.90% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 96.00% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 96.10% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 96.20% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 96.30% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 96.40% | train_error: 3.55 | train_acc: 0.829 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 96.50% | train_error: 3.50 | train_acc: 0.831 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 96.60% | train_error: 3.50 | train_acc: 0.831 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 96.70% | train_error: 3.50 | train_acc: 0.831 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 96.80% | train_error: 3.50 | train_acc: 0.831 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 96.90% | train_error: 3.50 | train_acc: 0.831 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 97.00% | train_error: 3.50 | train_acc: 0.831 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 97.10% | train_error: 3.50 | train_acc: 0.831 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 97.20% | train_error: 3.50 | train_acc: 0.831 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 97.30% | train_error: 3.50 | train_acc: 0.831 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [=====================================>--] 97.40% | train_error: 3.50 | train_acc: 0.831 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [======================================>-] 97.50% | train_error: 3.50 | train_acc: 0.831 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [======================================>-] 97.60% | train_error: 3.50 | train_acc: 0.831 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [======================================>-] 97.70% | train_error: 3.50 | train_acc: 0.831 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [======================================>-] 97.80% | train_error: 3.50 | train_acc: 0.831 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [======================================>-] 97.90% | train_error: 3.50 | train_acc: 0.831 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [======================================>-] 98.00% | train_error: 3.50 | train_acc: 0.831 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [======================================>-] 98.10% | train_error: 3.50 | train_acc: 0.831 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [======================================>-] 98.20% | train_error: 3.45 | train_acc: 0.833 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [======================================>-] 98.30% | train_error: 3.45 | train_acc: 0.833 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [======================================>-] 98.40% | train_error: 3.41 | train_acc: 0.836 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [======================================>-] 98.50% | train_error: 3.36 | train_acc: 0.838 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [======================================>-] 98.60% | train_error: 3.26 | train_acc: 0.843 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [======================================>-] 98.70% | train_error: 3.26 | train_acc: 0.843 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [======================================>-] 98.80% | train_error: 3.26 | train_acc: 0.843 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [======================================>-] 98.90% | train_error: 3.26 | train_acc: 0.843 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [======================================>-] 99.00% | train_error: 3.26 | train_acc: 0.843 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [======================================>-] 99.10% | train_error: 3.26 | train_acc: 0.843 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [======================================>-] 99.20% | train_error: 3.26 | train_acc: 0.843 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [======================================>-] 99.30% | train_error: 3.26 | train_acc: 0.843 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [======================================>-] 99.40% | train_error: 3.26 | train_acc: 0.843 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [======================================>-] 99.50% | train_error: 3.26 | train_acc: 0.843 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [======================================>-] 99.60% | train_error: 3.26 | train_acc: 0.843 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [======================================>-] 99.70% | train_error: 3.26 | train_acc: 0.843 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [======================================>-] 99.80% | train_error: 3.26 | train_acc: 0.843 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [======================================>-] 99.90% | train_error: 3.21 | train_acc: 0.845 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
                                                                                                                                  
+
    +
    +
      [=======================================>] 100.0% | train_error: 3.21 | train_acc: 0.845 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
    +
    +

    Finally, we will create a neural network with 2 hidden layers with activation functions.

    +
    +
    +
    input_nodes = X_train.shape[1]
+hidden_nodes1 = 100
+hidden_nodes2 = 30
+output_nodes = 1
+
+dims = (input_nodes, hidden_nodes1, hidden_nodes2, output_nodes)
+
+neural_network = FFNN(dims, hidden_func=RELU, output_func=sigmoid, cost_func=CostLogReg, seed=2023)
+
    +
    +
    +
    +
    +
    +
    neural_network.reset_weights() # reset weights such that previous runs or reruns don't affect the weights
+
+scheduler = Adam(eta=1e-4, rho=0.9, rho2=0.999)
+scores = neural_network.fit(X_train, t_train, scheduler, epochs=1000, X_val=X_val, t_val=t_val)
+
    +
    +
    +
    +
    Adam: Eta=0.0001, Lambda=0
+
    +
    +
      [----------------------------------------] 0.000% | train_error: 11.3 | train_acc: 0.453 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [----------------------------------------] 0.1000% | train_error: 11.4 | train_acc: 0.448 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [----------------------------------------] 0.2000% | train_error: 11.4 | train_acc: 0.448 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [----------------------------------------] 0.3000% | train_error: 11.4 | train_acc: 0.448 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [----------------------------------------] 0.4000% | train_error: 11.4 | train_acc: 0.448 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [----------------------------------------] 0.5000% | train_error: 11.4 | train_acc: 0.448 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [----------------------------------------] 0.6000% | train_error: 11.4 | train_acc: 0.448 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [----------------------------------------] 0.7000% | train_error: 11.5 | train_acc: 0.444 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [----------------------------------------] 0.8000% | train_error: 11.5 | train_acc: 0.444 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [----------------------------------------] 0.9000% | train_error: 11.3 | train_acc: 0.453 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [----------------------------------------] 1.000% | train_error: 11.3 | train_acc: 0.453 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [----------------------------------------] 1.100% | train_error: 11.3 | train_acc: 0.453 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [----------------------------------------] 1.200% | train_error: 11.3 | train_acc: 0.453 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [----------------------------------------] 1.300% | train_error: 11.3 | train_acc: 0.453 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [----------------------------------------] 1.400% | train_error: 11.4 | train_acc: 0.448 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [----------------------------------------] 1.500% | train_error: 11.4 | train_acc: 0.451 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [----------------------------------------] 1.600% | train_error: 11.5 | train_acc: 0.444 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [----------------------------------------] 1.700% | train_error: 11.5 | train_acc: 0.444 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [----------------------------------------] 1.800% | train_error: 11.6 | train_acc: 0.441 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [----------------------------------------] 1.900% | train_error: 11.6 | train_acc: 0.441 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [----------------------------------------] 2.000% | train_error: 11.6 | train_acc: 0.441 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [----------------------------------------] 2.100% | train_error: 11.6 | train_acc: 0.439 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [----------------------------------------] 2.200% | train_error: 11.6 | train_acc: 0.439 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [----------------------------------------] 2.300% | train_error: 11.8 | train_acc: 0.430 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [----------------------------------------] 2.400% | train_error: 11.9 | train_acc: 0.425 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [>---------------------------------------] 2.500% | train_error: 11.9 | train_acc: 0.425 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [>---------------------------------------] 2.600% | train_error: 11.9 | train_acc: 0.427 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [>---------------------------------------] 2.700% | train_error: 11.9 | train_acc: 0.427 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [>---------------------------------------] 2.800% | train_error: 11.9 | train_acc: 0.427 | val_error: 10.7 | val_acc: 0.483 
+
    +
    +
      [>---------------------------------------] 2.900% | train_error: 11.9 | train_acc: 0.427 | val_error: 10.7 | val_acc: 0.483 
+
    +
    +
      [>---------------------------------------] 3.000% | train_error: 11.9 | train_acc: 0.427 | val_error: 10.7 | val_acc: 0.483 
+
    +
    +
      [>---------------------------------------] 3.100% | train_error: 11.9 | train_acc: 0.427 | val_error: 10.7 | val_acc: 0.483 
+
    +
    +
      [>---------------------------------------] 3.200% | train_error: 11.9 | train_acc: 0.427 | val_error: 10.7 | val_acc: 0.483 
+
    +
    +
      [>---------------------------------------] 3.300% | train_error: 11.9 | train_acc: 0.425 | val_error: 10.9 | val_acc: 0.476 
+
    +
    +
      [>---------------------------------------] 3.400% | train_error: 11.9 | train_acc: 0.427 | val_error: 10.9 | val_acc: 0.476 
+
    +
    +
      [>---------------------------------------] 3.500% | train_error: 11.9 | train_acc: 0.427 | val_error: 11.0 | val_acc: 0.469 
+
    +
    +
      [>---------------------------------------] 3.600% | train_error: 11.8 | train_acc: 0.432 | val_error: 11.0 | val_acc: 0.469 
+
    +
    +
      [>---------------------------------------] 3.700% | train_error: 11.6 | train_acc: 0.439 | val_error: 11.0 | val_acc: 0.469 
+
    +
    +
      [>---------------------------------------] 3.800% | train_error: 11.6 | train_acc: 0.439 | val_error: 11.0 | val_acc: 0.469 
+
    +
    +
      [>---------------------------------------] 3.900% | train_error: 11.7 | train_acc: 0.434 | val_error: 11.3 | val_acc: 0.455 
+
    +
    +
      [>---------------------------------------] 4.000% | train_error: 11.7 | train_acc: 0.434 | val_error: 11.3 | val_acc: 0.455 
+
    +
    +
      [>---------------------------------------] 4.100% | train_error: 11.8 | train_acc: 0.430 | val_error: 11.2 | val_acc: 0.462 
+
    +
    +
      [>---------------------------------------] 4.200% | train_error: 11.9 | train_acc: 0.425 | val_error: 11.2 | val_acc: 0.462 
+
    +
    +
      [>---------------------------------------] 4.300% | train_error: 12.0 | train_acc: 0.423 | val_error: 11.2 | val_acc: 0.462 
+
    +
    +
      [>---------------------------------------] 4.400% | train_error: 11.9 | train_acc: 0.427 | val_error: 11.2 | val_acc: 0.462 
+
    +
    +
      [>---------------------------------------] 4.500% | train_error: 11.8 | train_acc: 0.430 | val_error: 11.2 | val_acc: 0.462 
+
    +
    +
      [>---------------------------------------] 4.600% | train_error: 11.8 | train_acc: 0.432 | val_error: 11.2 | val_acc: 0.462 
+
    +
    +
      [>---------------------------------------] 4.700% | train_error: 11.8 | train_acc: 0.430 | val_error: 11.3 | val_acc: 0.455 
+
    +
    +
      [>---------------------------------------] 4.800% | train_error: 11.7 | train_acc: 0.434 | val_error: 11.3 | val_acc: 0.455 
+
    +
    +
      [>---------------------------------------] 4.900% | train_error: 11.5 | train_acc: 0.446 | val_error: 11.3 | val_acc: 0.455 
+
    +
    +
      [=>--------------------------------------] 5.000% | train_error: 11.5 | train_acc: 0.446 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [=>--------------------------------------] 5.100% | train_error: 11.5 | train_acc: 0.446 | val_error: 11.3 | val_acc: 0.455 
+
    +
    +
      [=>--------------------------------------] 5.200% | train_error: 11.5 | train_acc: 0.446 | val_error: 11.2 | val_acc: 0.462 
+
    +
    +
      [=>--------------------------------------] 5.300% | train_error: 11.5 | train_acc: 0.444 | val_error: 11.0 | val_acc: 0.469 
+
    +
    +
      [=>--------------------------------------] 5.400% | train_error: 11.5 | train_acc: 0.444 | val_error: 11.0 | val_acc: 0.469 
+
    +
    +
      [=>--------------------------------------] 5.500% | train_error: 11.5 | train_acc: 0.444 | val_error: 11.0 | val_acc: 0.469 
+
    +
    +
      [=>--------------------------------------] 5.600% | train_error: 11.5 | train_acc: 0.444 | val_error: 11.3 | val_acc: 0.455 
+
    +
    +
      [=>--------------------------------------] 5.700% | train_error: 11.5 | train_acc: 0.444 | val_error: 11.3 | val_acc: 0.455 
+
    +
    +
      [=>--------------------------------------] 5.800% | train_error: 11.9 | train_acc: 0.425 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [=>--------------------------------------] 5.900% | train_error: 11.9 | train_acc: 0.425 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [=>--------------------------------------] 6.000% | train_error: 12.0 | train_acc: 0.420 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [=>--------------------------------------] 6.100% | train_error: 12.0 | train_acc: 0.420 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [=>--------------------------------------] 6.200% | train_error: 12.1 | train_acc: 0.418 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [=>--------------------------------------] 6.300% | train_error: 12.1 | train_acc: 0.418 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [=>--------------------------------------] 6.400% | train_error: 11.9 | train_acc: 0.425 | val_error: 11.7 | val_acc: 0.434 
+
    +
    +
      [=>--------------------------------------] 6.500% | train_error: 11.9 | train_acc: 0.427 | val_error: 11.9 | val_acc: 0.427 
+
    +
    +
      [=>--------------------------------------] 6.600% | train_error: 11.8 | train_acc: 0.432 | val_error: 12.0 | val_acc: 0.420 
+
    +
    +
      [=>--------------------------------------] 6.700% | train_error: 11.5 | train_acc: 0.444 | val_error: 12.2 | val_acc: 0.413 
+
    +
    +
      [=>--------------------------------------] 6.800% | train_error: 11.4 | train_acc: 0.448 | val_error: 12.3 | val_acc: 0.406 
+
    +
    +
      [=>--------------------------------------] 6.900% | train_error: 11.2 | train_acc: 0.458 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [=>--------------------------------------] 7.000% | train_error: 11.2 | train_acc: 0.458 | val_error: 12.3 | val_acc: 0.406 
+
    +
    +
      [=>--------------------------------------] 7.100% | train_error: 11.3 | train_acc: 0.455 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [=>--------------------------------------] 7.200% | train_error: 11.3 | train_acc: 0.455 | val_error: 12.6 | val_acc: 0.392 
+
    +
    +
      [=>--------------------------------------] 7.300% | train_error: 11.3 | train_acc: 0.455 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [=>--------------------------------------] 7.400% | train_error: 11.3 | train_acc: 0.453 | val_error: 12.3 | val_acc: 0.406 
+
    +
    +
      [==>-------------------------------------] 7.500% | train_error: 11.3 | train_acc: 0.455 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [==>-------------------------------------] 7.600% | train_error: 11.5 | train_acc: 0.446 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [==>-------------------------------------] 7.700% | train_error: 11.3 | train_acc: 0.453 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [==>-------------------------------------] 7.800% | train_error: 11.4 | train_acc: 0.448 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [==>-------------------------------------] 7.900% | train_error: 11.4 | train_acc: 0.448 | val_error: 12.3 | val_acc: 0.406 
+
    +
    +
      [==>-------------------------------------] 8.000% | train_error: 11.4 | train_acc: 0.451 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [==>-------------------------------------] 8.100% | train_error: 11.4 | train_acc: 0.448 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [==>-------------------------------------] 8.200% | train_error: 11.3 | train_acc: 0.453 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [==>-------------------------------------] 8.300% | train_error: 11.3 | train_acc: 0.453 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [==>-------------------------------------] 8.400% | train_error: 11.4 | train_acc: 0.451 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [==>-------------------------------------] 8.500% | train_error: 11.4 | train_acc: 0.451 | val_error: 12.5 | val_acc: 0.399 
+
    +
    +
      [==>-------------------------------------] 8.600% | train_error: 11.4 | train_acc: 0.451 | val_error: 12.3 | val_acc: 0.406 
+
    +
    +
      [==>-------------------------------------] 8.700% | train_error: 11.2 | train_acc: 0.458 | val_error: 12.3 | val_acc: 0.406 
+
    +
    +
      [==>-------------------------------------] 8.800% | train_error: 11.2 | train_acc: 0.458 | val_error: 12.2 | val_acc: 0.413 
+
    +
    +
      [==>-------------------------------------] 8.900% | train_error: 11.2 | train_acc: 0.458 | val_error: 12.0 | val_acc: 0.420 
+
    +
    +
      [==>-------------------------------------] 9.000% | train_error: 11.2 | train_acc: 0.458 | val_error: 12.0 | val_acc: 0.420 
+
    +
    +
      [==>-------------------------------------] 9.100% | train_error: 11.2 | train_acc: 0.458 | val_error: 11.9 | val_acc: 0.427 
+
    +
    +
      [==>-------------------------------------] 9.200% | train_error: 11.2 | train_acc: 0.458 | val_error: 11.9 | val_acc: 0.427 
+
    +
    +
      [==>-------------------------------------] 9.300% | train_error: 11.2 | train_acc: 0.460 | val_error: 11.9 | val_acc: 0.427 
+
    +
    +
      [==>-------------------------------------] 9.400% | train_error: 11.0 | train_acc: 0.469 | val_error: 11.9 | val_acc: 0.427 
+
    +
    +
      [==>-------------------------------------] 9.500% | train_error: 11.0 | train_acc: 0.469 | val_error: 11.9 | val_acc: 0.427 
+
    +
    +
      [==>-------------------------------------] 9.600% | train_error: 10.8 | train_acc: 0.479 | val_error: 11.9 | val_acc: 0.427 
+
    +
    +
      [==>-------------------------------------] 9.700% | train_error: 10.7 | train_acc: 0.484 | val_error: 11.7 | val_acc: 0.434 
+
    +
    +
      [==>-------------------------------------] 9.800% | train_error: 10.7 | train_acc: 0.484 | val_error: 11.7 | val_acc: 0.434 
+
    +
    +
      [==>-------------------------------------] 9.900% | train_error: 10.7 | train_acc: 0.484 | val_error: 11.7 | val_acc: 0.434 
+
    +
    +
      [===>------------------------------------] 10.00% | train_error: 10.7 | train_acc: 0.484 | val_error: 11.7 | val_acc: 0.434 
+
    +
    +
      [===>------------------------------------] 10.10% | train_error: 10.7 | train_acc: 0.486 | val_error: 11.7 | val_acc: 0.434 
+
    +
    +
      [===>------------------------------------] 10.20% | train_error: 10.7 | train_acc: 0.486 | val_error: 11.6 | val_acc: 0.441 
+
    +
    +
      [===>------------------------------------] 10.30% | train_error: 10.7 | train_acc: 0.486 | val_error: 11.6 | val_acc: 0.441 
+
    +
    +
      [===>------------------------------------] 10.40% | train_error: 10.7 | train_acc: 0.486 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [===>------------------------------------] 10.50% | train_error: 10.4 | train_acc: 0.500 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [===>------------------------------------] 10.60% | train_error: 10.4 | train_acc: 0.500 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [===>------------------------------------] 10.70% | train_error: 10.4 | train_acc: 0.500 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [===>------------------------------------] 10.80% | train_error: 10.4 | train_acc: 0.500 | val_error: 11.4 | val_acc: 0.448 
+
    +
    +
      [===>------------------------------------] 10.90% | train_error: 10.2 | train_acc: 0.507 | val_error: 11.3 | val_acc: 0.455 
+
    +
    +
      [===>------------------------------------] 11.00% | train_error: 9.83 | train_acc: 0.526 | val_error: 11.3 | val_acc: 0.455 
+
    +
    +
      [===>------------------------------------] 11.10% | train_error: 9.83 | train_acc: 0.526 | val_error: 11.3 | val_acc: 0.455 
+
    +
    +
      [===>------------------------------------] 11.20% | train_error: 9.83 | train_acc: 0.526 | val_error: 11.3 | val_acc: 0.455 
+
    +
    +
      [===>------------------------------------] 11.30% | train_error: 9.83 | train_acc: 0.526 | val_error: 11.2 | val_acc: 0.462 
+
    +
    +
      [===>------------------------------------] 11.40% | train_error: 9.83 | train_acc: 0.526 | val_error: 11.2 | val_acc: 0.462 
+
    +
    +
      [===>------------------------------------] 11.50% | train_error: 9.83 | train_acc: 0.526 | val_error: 11.2 | val_acc: 0.462 
+
    +
    +
      [===>------------------------------------] 11.60% | train_error: 9.83 | train_acc: 0.526 | val_error: 11.2 | val_acc: 0.462 
+
    +
    +
      [===>------------------------------------] 11.70% | train_error: 9.73 | train_acc: 0.531 | val_error: 11.2 | val_acc: 0.462 
+
    +
    +
      [===>------------------------------------] 11.80% | train_error: 9.63 | train_acc: 0.535 | val_error: 11.2 | val_acc: 0.462 
+
    +
    +
      [===>------------------------------------] 11.90% | train_error: 9.63 | train_acc: 0.535 | val_error: 11.2 | val_acc: 0.462 
+
    +
    +
      [===>------------------------------------] 12.00% | train_error: 9.53 | train_acc: 0.540 | val_error: 10.9 | val_acc: 0.476 
+
    +
    +
      [===>------------------------------------] 12.10% | train_error: 9.53 | train_acc: 0.540 | val_error: 10.9 | val_acc: 0.476 
+
    +
    +
      [===>------------------------------------] 12.20% | train_error: 9.49 | train_acc: 0.542 | val_error: 10.9 | val_acc: 0.476 
+
    +
    +
      [===>------------------------------------] 12.30% | train_error: 9.49 | train_acc: 0.542 | val_error: 10.9 | val_acc: 0.476 
+
    +
    +
      [===>------------------------------------] 12.40% | train_error: 9.44 | train_acc: 0.545 | val_error: 10.9 | val_acc: 0.476 
+
    +
    +
      [====>-----------------------------------] 12.50% | train_error: 9.44 | train_acc: 0.545 | val_error: 10.9 | val_acc: 0.476 
+
    +
    +
      [====>-----------------------------------] 12.60% | train_error: 9.39 | train_acc: 0.547 | val_error: 10.9 | val_acc: 0.476 
+
    +
    +
      [====>-----------------------------------] 12.70% | train_error: 9.39 | train_acc: 0.547 | val_error: 10.7 | val_acc: 0.483 
+
    +
    +
      [====>-----------------------------------] 12.80% | train_error: 9.39 | train_acc: 0.547 | val_error: 10.7 | val_acc: 0.483 
+
    +
    +
      [====>-----------------------------------] 12.90% | train_error: 9.34 | train_acc: 0.549 | val_error: 10.7 | val_acc: 0.483 
+
    +
    +
      [====>-----------------------------------] 13.00% | train_error: 9.10 | train_acc: 0.561 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [====>-----------------------------------] 13.10% | train_error: 9.05 | train_acc: 0.563 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [====>-----------------------------------] 13.20% | train_error: 9.05 | train_acc: 0.563 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [====>-----------------------------------] 13.30% | train_error: 8.80 | train_acc: 0.575 | val_error: 10.6 | val_acc: 0.490 
+
    +
    +
      [====>-----------------------------------] 13.40% | train_error: 8.76 | train_acc: 0.577 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [====>-----------------------------------] 13.50% | train_error: 8.56 | train_acc: 0.587 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [====>-----------------------------------] 13.60% | train_error: 8.56 | train_acc: 0.587 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [====>-----------------------------------] 13.70% | train_error: 8.42 | train_acc: 0.594 | val_error: 10.4 | val_acc: 0.497 
+
    +
    +
      [====>-----------------------------------] 13.80% | train_error: 8.42 | train_acc: 0.594 | val_error: 10.3 | val_acc: 0.503 
+
    +
    +
      [====>-----------------------------------] 13.90% | train_error: 8.42 | train_acc: 0.594 | val_error: 10.1 | val_acc: 0.510 
+
    +
    +
      [====>-----------------------------------] 14.00% | train_error: 8.42 | train_acc: 0.594 | val_error: 10.1 | val_acc: 0.510 
+
    +
    +
      [====>-----------------------------------] 14.10% | train_error: 8.32 | train_acc: 0.599 | val_error: 10.1 | val_acc: 0.510 
+
    +
    +
      [====>-----------------------------------] 14.20% | train_error: 8.32 | train_acc: 0.599 | val_error: 10.1 | val_acc: 0.510 
+
    +
    +
      [====>-----------------------------------] 14.30% | train_error: 8.22 | train_acc: 0.603 | val_error: 10.1 | val_acc: 0.510 
+
    +
    +
      [====>-----------------------------------] 14.40% | train_error: 8.22 | train_acc: 0.603 | val_error: 10.1 | val_acc: 0.510 
+
    +
    +
      [====>-----------------------------------] 14.50% | train_error: 8.22 | train_acc: 0.603 | val_error: 10.1 | val_acc: 0.510 
+
    +
    +
      [====>-----------------------------------] 14.60% | train_error: 8.22 | train_acc: 0.603 | val_error: 10.1 | val_acc: 0.510 
+
    +
    +
      [====>-----------------------------------] 14.70% | train_error: 8.17 | train_acc: 0.606 | val_error: 10.1 | val_acc: 0.510 
+
    +
    +
      [====>-----------------------------------] 14.80% | train_error: 8.17 | train_acc: 0.606 | val_error: 10.1 | val_acc: 0.510 
+
    +
    +
      [====>-----------------------------------] 14.90% | train_error: 8.17 | train_acc: 0.606 | val_error: 10.1 | val_acc: 0.510 
+
    +
    +
      [=====>----------------------------------] 15.00% | train_error: 8.08 | train_acc: 0.610 | val_error: 9.85 | val_acc: 0.524 
+
    +
    +
      [=====>----------------------------------] 15.10% | train_error: 7.98 | train_acc: 0.615 | val_error: 9.71 | val_acc: 0.531 
+
    +
    +
      [=====>----------------------------------] 15.20% | train_error: 7.93 | train_acc: 0.617 | val_error: 9.71 | val_acc: 0.531 
+
    +
    +
      [=====>----------------------------------] 15.30% | train_error: 7.93 | train_acc: 0.617 | val_error: 9.71 | val_acc: 0.531 
+
    +
    +
      [=====>----------------------------------] 15.40% | train_error: 7.93 | train_acc: 0.617 | val_error: 9.71 | val_acc: 0.531 
+
    +
    +
      [=====>----------------------------------] 15.50% | train_error: 7.93 | train_acc: 0.617 | val_error: 9.71 | val_acc: 0.531 
+
    +
    +
      [=====>----------------------------------] 15.60% | train_error: 7.93 | train_acc: 0.617 | val_error: 9.71 | val_acc: 0.531 
+
    +
    +
      [=====>----------------------------------] 15.70% | train_error: 7.93 | train_acc: 0.617 | val_error: 9.71 | val_acc: 0.531 
+
    +
    +
      [=====>----------------------------------] 15.80% | train_error: 7.93 | train_acc: 0.617 | val_error: 9.56 | val_acc: 0.538 
+
    +
    +
      [=====>----------------------------------] 15.90% | train_error: 7.93 | train_acc: 0.617 | val_error: 9.27 | val_acc: 0.552 
+
    +
    +
      [=====>----------------------------------] 16.00% | train_error: 7.93 | train_acc: 0.617 | val_error: 9.27 | val_acc: 0.552 
+
    +
    +
      [=====>----------------------------------] 16.10% | train_error: 7.93 | train_acc: 0.617 | val_error: 9.27 | val_acc: 0.552 
+
    +
    +
      [=====>----------------------------------] 16.20% | train_error: 7.88 | train_acc: 0.620 | val_error: 9.27 | val_acc: 0.552 
+
    +
    +
      [=====>----------------------------------] 16.30% | train_error: 7.69 | train_acc: 0.629 | val_error: 9.27 | val_acc: 0.552 
+
    +
    +
      [=====>----------------------------------] 16.40% | train_error: 7.69 | train_acc: 0.629 | val_error: 9.13 | val_acc: 0.559 
+
    +
    +
      [=====>----------------------------------] 16.50% | train_error: 7.69 | train_acc: 0.629 | val_error: 9.13 | val_acc: 0.559 
+
    +
    +
      [=====>----------------------------------] 16.60% | train_error: 7.69 | train_acc: 0.629 | val_error: 9.13 | val_acc: 0.559 
+
    +
    +
      [=====>----------------------------------] 16.70% | train_error: 7.69 | train_acc: 0.629 | val_error: 8.98 | val_acc: 0.566 
+
    +
    +
      [=====>----------------------------------] 16.80% | train_error: 7.59 | train_acc: 0.634 | val_error: 8.84 | val_acc: 0.573 
+
    +
    +
      [=====>----------------------------------] 16.90% | train_error: 7.49 | train_acc: 0.638 | val_error: 8.84 | val_acc: 0.573 
+
    +
    +
      [=====>----------------------------------] 17.00% | train_error: 7.39 | train_acc: 0.643 | val_error: 8.70 | val_acc: 0.580 
+
    +
    +
      [=====>----------------------------------] 17.10% | train_error: 7.39 | train_acc: 0.643 | val_error: 8.55 | val_acc: 0.587 
+
    +
    +
      [=====>----------------------------------] 17.20% | train_error: 7.30 | train_acc: 0.648 | val_error: 8.41 | val_acc: 0.594 
+
    +
    +
      [=====>----------------------------------] 17.30% | train_error: 7.30 | train_acc: 0.648 | val_error: 8.26 | val_acc: 0.601 
+
    +
    +
      [=====>----------------------------------] 17.40% | train_error: 7.15 | train_acc: 0.655 | val_error: 8.26 | val_acc: 0.601 
+
    +
    +
      [======>---------------------------------] 17.50% | train_error: 7.05 | train_acc: 0.660 | val_error: 8.12 | val_acc: 0.608 
+
    +
    +
      [======>---------------------------------] 17.60% | train_error: 7.05 | train_acc: 0.660 | val_error: 7.97 | val_acc: 0.615 
+
    +
    +
      [======>---------------------------------] 17.70% | train_error: 6.91 | train_acc: 0.667 | val_error: 7.97 | val_acc: 0.615 
+
    +
    +
      [======>---------------------------------] 17.80% | train_error: 6.86 | train_acc: 0.669 | val_error: 7.83 | val_acc: 0.622 
+
    +
    +
      [======>---------------------------------] 17.90% | train_error: 6.86 | train_acc: 0.669 | val_error: 7.83 | val_acc: 0.622 
+
    +
    +
      [======>---------------------------------] 18.00% | train_error: 6.86 | train_acc: 0.669 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [======>---------------------------------] 18.10% | train_error: 6.76 | train_acc: 0.674 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [======>---------------------------------] 18.20% | train_error: 6.76 | train_acc: 0.674 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [======>---------------------------------] 18.30% | train_error: 6.62 | train_acc: 0.681 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [======>---------------------------------] 18.40% | train_error: 6.57 | train_acc: 0.683 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [======>---------------------------------] 18.50% | train_error: 6.47 | train_acc: 0.688 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [======>---------------------------------] 18.60% | train_error: 6.47 | train_acc: 0.688 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [======>---------------------------------] 18.70% | train_error: 6.37 | train_acc: 0.692 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [======>---------------------------------] 18.80% | train_error: 6.32 | train_acc: 0.695 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [======>---------------------------------] 18.90% | train_error: 6.28 | train_acc: 0.697 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [======>---------------------------------] 19.00% | train_error: 6.28 | train_acc: 0.697 | val_error: 7.54 | val_acc: 0.636 
+
    +
    +
      [======>---------------------------------] 19.10% | train_error: 6.28 | train_acc: 0.697 | val_error: 7.39 | val_acc: 0.643 
+
    +
    +
      [======>---------------------------------] 19.20% | train_error: 6.28 | train_acc: 0.697 | val_error: 7.39 | val_acc: 0.643 
+
    +
    +
      [======>---------------------------------] 19.30% | train_error: 6.28 | train_acc: 0.697 | val_error: 7.10 | val_acc: 0.657 
+
    +
    +
      [======>---------------------------------] 19.40% | train_error: 6.18 | train_acc: 0.702 | val_error: 7.10 | val_acc: 0.657 
+
    +
    +
      [======>---------------------------------] 19.50% | train_error: 6.18 | train_acc: 0.702 | val_error: 6.81 | val_acc: 0.671 
+
    +
    +
      [======>---------------------------------] 19.60% | train_error: 6.13 | train_acc: 0.704 | val_error: 6.81 | val_acc: 0.671 
+
    +
    +
      [======>---------------------------------] 19.70% | train_error: 6.13 | train_acc: 0.704 | val_error: 6.67 | val_acc: 0.678 
+
    +
    +
      [======>---------------------------------] 19.80% | train_error: 6.13 | train_acc: 0.704 | val_error: 6.52 | val_acc: 0.685 
+
    +
    +
      [======>---------------------------------] 19.90% | train_error: 6.13 | train_acc: 0.704 | val_error: 6.52 | val_acc: 0.685 
+
    +
    +
      [=======>--------------------------------] 20.00% | train_error: 6.13 | train_acc: 0.704 | val_error: 6.52 | val_acc: 0.685 
+
    +
    +
      [=======>--------------------------------] 20.10% | train_error: 6.08 | train_acc: 0.707 | val_error: 6.38 | val_acc: 0.692 
+
    +
    +
      [=======>--------------------------------] 20.20% | train_error: 6.03 | train_acc: 0.709 | val_error: 6.52 | val_acc: 0.685 
+
    +
    +
      [=======>--------------------------------] 20.30% | train_error: 5.64 | train_acc: 0.728 | val_error: 6.52 | val_acc: 0.685 
+
    +
    +
      [=======>--------------------------------] 20.40% | train_error: 5.69 | train_acc: 0.725 | val_error: 6.52 | val_acc: 0.685 
+
    +
    +
      [=======>--------------------------------] 20.50% | train_error: 5.69 | train_acc: 0.725 | val_error: 6.52 | val_acc: 0.685 
+
    +
    +
      [=======>--------------------------------] 20.60% | train_error: 5.59 | train_acc: 0.730 | val_error: 6.52 | val_acc: 0.685 
+
    +
    +
      [=======>--------------------------------] 20.70% | train_error: 5.59 | train_acc: 0.730 | val_error: 6.67 | val_acc: 0.678 
+
    +
    +
      [=======>--------------------------------] 20.80% | train_error: 5.55 | train_acc: 0.732 | val_error: 6.67 | val_acc: 0.678 
+
    +
    +
      [=======>--------------------------------] 20.90% | train_error: 5.50 | train_acc: 0.735 | val_error: 6.52 | val_acc: 0.685 
+
    +
    +
      [=======>--------------------------------] 21.00% | train_error: 5.35 | train_acc: 0.742 | val_error: 6.38 | val_acc: 0.692 
+
    +
    +
      [=======>--------------------------------] 21.10% | train_error: 5.35 | train_acc: 0.742 | val_error: 6.23 | val_acc: 0.699 
+
    +
    +
      [=======>--------------------------------] 21.20% | train_error: 5.30 | train_acc: 0.744 | val_error: 6.23 | val_acc: 0.699 
+
    +
    +
      [=======>--------------------------------] 21.30% | train_error: 5.21 | train_acc: 0.749 | val_error: 6.23 | val_acc: 0.699 
+
    +
    +
      [=======>--------------------------------] 21.40% | train_error: 5.06 | train_acc: 0.756 | val_error: 6.23 | val_acc: 0.699 
+
    +
    +
      [=======>--------------------------------] 21.50% | train_error: 5.06 | train_acc: 0.756 | val_error: 6.09 | val_acc: 0.706 
+
    +
    +
      [=======>--------------------------------] 21.60% | train_error: 5.06 | train_acc: 0.756 | val_error: 6.09 | val_acc: 0.706 
+
    +
    +
      [=======>--------------------------------] 21.70% | train_error: 5.01 | train_acc: 0.758 | val_error: 6.09 | val_acc: 0.706 
+
    +
    +
      [=======>--------------------------------] 21.80% | train_error: 5.01 | train_acc: 0.758 | val_error: 6.09 | val_acc: 0.706 
+
    +
    +
      [=======>--------------------------------] 21.90% | train_error: 4.96 | train_acc: 0.761 | val_error: 6.09 | val_acc: 0.706 
+
    +
    +
      [=======>--------------------------------] 22.00% | train_error: 4.96 | train_acc: 0.761 | val_error: 5.65 | val_acc: 0.727 
+
    +
    +
      [=======>--------------------------------] 22.10% | train_error: 4.72 | train_acc: 0.772 | val_error: 5.65 | val_acc: 0.727 
+
    +
    +
      [=======>--------------------------------] 22.20% | train_error: 4.62 | train_acc: 0.777 | val_error: 5.65 | val_acc: 0.727 
+
    +
    +
      [=======>--------------------------------] 22.30% | train_error: 4.52 | train_acc: 0.782 | val_error: 5.65 | val_acc: 0.727 
+
    +
    +
      [=======>--------------------------------] 22.40% | train_error: 4.62 | train_acc: 0.777 | val_error: 5.65 | val_acc: 0.727 
+
    +
    +
      [========>-------------------------------] 22.50% | train_error: 4.52 | train_acc: 0.782 | val_error: 5.51 | val_acc: 0.734 
+
    +
    +
      [========>-------------------------------] 22.60% | train_error: 4.57 | train_acc: 0.779 | val_error: 5.65 | val_acc: 0.727 
+
    +
    +
      [========>-------------------------------] 22.70% | train_error: 4.52 | train_acc: 0.782 | val_error: 5.65 | val_acc: 0.727 
+
    +
    +
      [========>-------------------------------] 22.80% | train_error: 4.33 | train_acc: 0.791 | val_error: 5.51 | val_acc: 0.734 
+
    +
    +
      [========>-------------------------------] 22.90% | train_error: 4.33 | train_acc: 0.791 | val_error: 5.36 | val_acc: 0.741 
+
    +
    +
      [========>-------------------------------] 23.00% | train_error: 4.33 | train_acc: 0.791 | val_error: 5.36 | val_acc: 0.741 
+
    +
    +
      [========>-------------------------------] 23.10% | train_error: 4.33 | train_acc: 0.791 | val_error: 5.36 | val_acc: 0.741 
+
    +
    +
      [========>-------------------------------] 23.20% | train_error: 4.33 | train_acc: 0.791 | val_error: 5.07 | val_acc: 0.755 
+
    +
    +
      [========>-------------------------------] 23.30% | train_error: 4.33 | train_acc: 0.791 | val_error: 5.07 | val_acc: 0.755 
+
    +
    +
      [========>-------------------------------] 23.40% | train_error: 4.28 | train_acc: 0.793 | val_error: 5.07 | val_acc: 0.755 
+
    +
    +
      [========>-------------------------------] 23.50% | train_error: 4.28 | train_acc: 0.793 | val_error: 5.07 | val_acc: 0.755 
+
    +
    +
      [========>-------------------------------] 23.60% | train_error: 4.13 | train_acc: 0.800 | val_error: 5.07 | val_acc: 0.755 
+
    +
    +
      [========>-------------------------------] 23.70% | train_error: 4.04 | train_acc: 0.805 | val_error: 4.93 | val_acc: 0.762 
+
    +
    +
      [========>-------------------------------] 23.80% | train_error: 4.04 | train_acc: 0.805 | val_error: 4.78 | val_acc: 0.769 
+
    +
    +
      [========>-------------------------------] 23.90% | train_error: 4.04 | train_acc: 0.805 | val_error: 4.78 | val_acc: 0.769 
+
    +
    +
      [========>-------------------------------] 24.00% | train_error: 3.89 | train_acc: 0.812 | val_error: 4.64 | val_acc: 0.776 
+
    +
    +
      [========>-------------------------------] 24.10% | train_error: 3.84 | train_acc: 0.815 | val_error: 4.64 | val_acc: 0.776 
+
    +
    +
      [========>-------------------------------] 24.20% | train_error: 3.79 | train_acc: 0.817 | val_error: 4.64 | val_acc: 0.776 
+
    +
    +
      [========>-------------------------------] 24.30% | train_error: 3.70 | train_acc: 0.822 | val_error: 4.64 | val_acc: 0.776 
+
    +
    +
      [========>-------------------------------] 24.40% | train_error: 3.70 | train_acc: 0.822 | val_error: 4.49 | val_acc: 0.783 
+
    +
    +
      [========>-------------------------------] 24.50% | train_error: 3.65 | train_acc: 0.824 | val_error: 4.35 | val_acc: 0.790 
+
    +
    +
      [========>-------------------------------] 24.60% | train_error: 3.55 | train_acc: 0.829 | val_error: 4.35 | val_acc: 0.790 
+
    +
    +
      [========>-------------------------------] 24.70% | train_error: 3.55 | train_acc: 0.829 | val_error: 4.35 | val_acc: 0.790 
+
    +
    +
      [========>-------------------------------] 24.80% | train_error: 3.50 | train_acc: 0.831 | val_error: 4.64 | val_acc: 0.776 
+
    +
    +
      [========>-------------------------------] 24.90% | train_error: 3.45 | train_acc: 0.833 | val_error: 4.64 | val_acc: 0.776 
+
    +
    +
      [=========>------------------------------] 25.00% | train_error: 3.21 | train_acc: 0.845 | val_error: 4.35 | val_acc: 0.790 
+
    +
    +
      [=========>------------------------------] 25.10% | train_error: 3.21 | train_acc: 0.845 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [=========>------------------------------] 25.20% | train_error: 3.11 | train_acc: 0.850 | val_error: 4.20 | val_acc: 0.797 
+
    +
    +
      [=========>------------------------------] 25.30% | train_error: 3.11 | train_acc: 0.850 | val_error: 4.35 | val_acc: 0.790 
+
    +
    +
      [=========>------------------------------] 25.40% | train_error: 3.11 | train_acc: 0.850 | val_error: 4.06 | val_acc: 0.804 
+
    +
    +
      [=========>------------------------------] 25.50% | train_error: 3.11 | train_acc: 0.850 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [=========>------------------------------] 25.60% | train_error: 3.11 | train_acc: 0.850 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [=========>------------------------------] 25.70% | train_error: 3.11 | train_acc: 0.850 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [=========>------------------------------] 25.80% | train_error: 3.11 | train_acc: 0.850 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [=========>------------------------------] 25.90% | train_error: 3.11 | train_acc: 0.850 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [=========>------------------------------] 26.00% | train_error: 3.21 | train_acc: 0.845 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [=========>------------------------------] 26.10% | train_error: 3.21 | train_acc: 0.845 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [=========>------------------------------] 26.20% | train_error: 3.21 | train_acc: 0.845 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [=========>------------------------------] 26.30% | train_error: 3.21 | train_acc: 0.845 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [=========>------------------------------] 26.40% | train_error: 3.21 | train_acc: 0.845 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [=========>------------------------------] 26.50% | train_error: 3.02 | train_acc: 0.854 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [=========>------------------------------] 26.60% | train_error: 2.77 | train_acc: 0.866 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [=========>------------------------------] 26.70% | train_error: 2.77 | train_acc: 0.866 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [=========>------------------------------] 26.80% | train_error: 2.72 | train_acc: 0.869 | val_error: 3.91 | val_acc: 0.811 
+
    +
    +
      [=========>------------------------------] 26.90% | train_error: 2.68 | train_acc: 0.871 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [=========>------------------------------] 27.00% | train_error: 2.53 | train_acc: 0.878 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [=========>------------------------------] 27.10% | train_error: 2.48 | train_acc: 0.880 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [=========>------------------------------] 27.20% | train_error: 2.48 | train_acc: 0.880 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [=========>------------------------------] 27.30% | train_error: 2.38 | train_acc: 0.885 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [=========>------------------------------] 27.40% | train_error: 2.38 | train_acc: 0.885 | val_error: 3.77 | val_acc: 0.818 
+
    +
    +
      [==========>-----------------------------] 27.50% | train_error: 2.38 | train_acc: 0.885 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [==========>-----------------------------] 27.60% | train_error: 2.29 | train_acc: 0.890 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [==========>-----------------------------] 27.70% | train_error: 2.24 | train_acc: 0.892 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [==========>-----------------------------] 27.80% | train_error: 2.24 | train_acc: 0.892 | val_error: 3.62 | val_acc: 0.825 
+
    +
    +
      [==========>-----------------------------] 27.90% | train_error: 2.19 | train_acc: 0.894 | val_error: 3.48 | val_acc: 0.832 
+
    +
    +
      [==========>-----------------------------] 28.00% | train_error: 2.19 | train_acc: 0.894 | val_error: 3.33 | val_acc: 0.839 
+
    +
    +
      [==========>-----------------------------] 28.10% | train_error: 2.19 | train_acc: 0.894 | val_error: 3.19 | val_acc: 0.846 
+
    +
    +
      [==========>-----------------------------] 28.20% | train_error: 2.19 | train_acc: 0.894 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [==========>-----------------------------] 28.30% | train_error: 2.19 | train_acc: 0.894 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [==========>-----------------------------] 28.40% | train_error: 2.14 | train_acc: 0.897 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [==========>-----------------------------] 28.50% | train_error: 2.09 | train_acc: 0.899 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [==========>-----------------------------] 28.60% | train_error: 2.04 | train_acc: 0.901 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [==========>-----------------------------] 28.70% | train_error: 2.09 | train_acc: 0.899 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [==========>-----------------------------] 28.80% | train_error: 2.09 | train_acc: 0.899 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [==========>-----------------------------] 28.90% | train_error: 2.09 | train_acc: 0.899 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [==========>-----------------------------] 29.00% | train_error: 2.09 | train_acc: 0.899 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [==========>-----------------------------] 29.10% | train_error: 2.09 | train_acc: 0.899 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [==========>-----------------------------] 29.20% | train_error: 2.09 | train_acc: 0.899 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [==========>-----------------------------] 29.30% | train_error: 2.04 | train_acc: 0.901 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [==========>-----------------------------] 29.40% | train_error: 2.04 | train_acc: 0.901 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [==========>-----------------------------] 29.50% | train_error: 2.04 | train_acc: 0.901 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [==========>-----------------------------] 29.60% | train_error: 2.04 | train_acc: 0.901 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [==========>-----------------------------] 29.70% | train_error: 2.04 | train_acc: 0.901 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [==========>-----------------------------] 29.80% | train_error: 1.99 | train_acc: 0.904 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [==========>-----------------------------] 29.90% | train_error: 1.95 | train_acc: 0.906 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [===========>----------------------------] 30.00% | train_error: 1.90 | train_acc: 0.908 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [===========>----------------------------] 30.10% | train_error: 1.90 | train_acc: 0.908 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [===========>----------------------------] 30.20% | train_error: 1.90 | train_acc: 0.908 | val_error: 3.19 | val_acc: 0.846 
+
    +
    +
      [===========>----------------------------] 30.30% | train_error: 1.90 | train_acc: 0.908 | val_error: 3.19 | val_acc: 0.846 
+
    +
    +
      [===========>----------------------------] 30.40% | train_error: 1.90 | train_acc: 0.908 | val_error: 3.19 | val_acc: 0.846 
+
    +
    +
      [===========>----------------------------] 30.50% | train_error: 1.90 | train_acc: 0.908 | val_error: 3.19 | val_acc: 0.846 
+
    +
    +
      [===========>----------------------------] 30.60% | train_error: 1.90 | train_acc: 0.908 | val_error: 3.19 | val_acc: 0.846 
+
    +
    +
      [===========>----------------------------] 30.70% | train_error: 1.90 | train_acc: 0.908 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [===========>----------------------------] 30.80% | train_error: 1.90 | train_acc: 0.908 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [===========>----------------------------] 30.90% | train_error: 1.90 | train_acc: 0.908 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [===========>----------------------------] 31.00% | train_error: 1.75 | train_acc: 0.915 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [===========>----------------------------] 31.10% | train_error: 1.75 | train_acc: 0.915 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [===========>----------------------------] 31.20% | train_error: 1.61 | train_acc: 0.923 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [===========>----------------------------] 31.30% | train_error: 1.61 | train_acc: 0.923 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [===========>----------------------------] 31.40% | train_error: 1.56 | train_acc: 0.925 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [===========>----------------------------] 31.50% | train_error: 1.56 | train_acc: 0.925 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [===========>----------------------------] 31.60% | train_error: 1.56 | train_acc: 0.925 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [===========>----------------------------] 31.70% | train_error: 1.51 | train_acc: 0.927 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [===========>----------------------------] 31.80% | train_error: 1.51 | train_acc: 0.927 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [===========>----------------------------] 31.90% | train_error: 1.51 | train_acc: 0.927 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [===========>----------------------------] 32.00% | train_error: 1.51 | train_acc: 0.927 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [===========>----------------------------] 32.10% | train_error: 1.41 | train_acc: 0.932 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [===========>----------------------------] 32.20% | train_error: 1.41 | train_acc: 0.932 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [===========>----------------------------] 32.30% | train_error: 1.41 | train_acc: 0.932 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [===========>----------------------------] 32.40% | train_error: 1.41 | train_acc: 0.932 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [============>---------------------------] 32.50% | train_error: 1.41 | train_acc: 0.932 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [============>---------------------------] 32.60% | train_error: 1.41 | train_acc: 0.932 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [============>---------------------------] 32.70% | train_error: 1.41 | train_acc: 0.932 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [============>---------------------------] 32.80% | train_error: 1.31 | train_acc: 0.937 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [============>---------------------------] 32.90% | train_error: 1.26 | train_acc: 0.939 | val_error: 3.04 | val_acc: 0.853 
+
    +
    +
      [============>---------------------------] 33.00% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [============>---------------------------] 33.10% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [============>---------------------------] 33.20% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [============>---------------------------] 33.30% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.90 | val_acc: 0.860 
+
    +
    +
      [============>---------------------------] 33.40% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.75 | val_acc: 0.867 
+
    +
    +
      [============>---------------------------] 33.50% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.75 | val_acc: 0.867 
+
    +
    +
      [============>---------------------------] 33.60% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.75 | val_acc: 0.867 
+
    +
    +
      [============>---------------------------] 33.70% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.75 | val_acc: 0.867 
+
    +
    +
      [============>---------------------------] 33.80% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.75 | val_acc: 0.867 
+
    +
    +
      [============>---------------------------] 33.90% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.75 | val_acc: 0.867 
+
    +
    +
      [============>---------------------------] 34.00% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.61 | val_acc: 0.874 
+
    +
    +
      [============>---------------------------] 34.10% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.61 | val_acc: 0.874 
+
    +
    +
      [============>---------------------------] 34.20% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.61 | val_acc: 0.874 
+
    +
    +
      [============>---------------------------] 34.30% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.61 | val_acc: 0.874 
+
    +
    +
      [============>---------------------------] 34.40% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.61 | val_acc: 0.874 
+
    +
    +
      [============>---------------------------] 34.50% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.75 | val_acc: 0.867 
+
    +
    +
      [============>---------------------------] 34.60% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.75 | val_acc: 0.867 
+
    +
    +
      [============>---------------------------] 34.70% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.75 | val_acc: 0.867 
+
    +
    +
      [============>---------------------------] 34.80% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.75 | val_acc: 0.867 
+
    +
    +
      [============>---------------------------] 34.90% | train_error: 1.22 | train_acc: 0.941 | val_error: 2.75 | val_acc: 0.867 
+
    +
    +
      [=============>--------------------------] 35.00% | train_error: 1.02 | train_acc: 0.951 | val_error: 2.75 | val_acc: 0.867 
+
    +
    +
      [=============>--------------------------] 35.10% | train_error: 1.02 | train_acc: 0.951 | val_error: 2.75 | val_acc: 0.867 
+
    +
    +
      [=============>--------------------------] 35.20% | train_error: 1.02 | train_acc: 0.951 | val_error: 2.75 | val_acc: 0.867 
+
    +
    +
      [=============>--------------------------] 35.30% | train_error: 1.02 | train_acc: 0.951 | val_error: 2.61 | val_acc: 0.874 
+
    +
    +
      [=============>--------------------------] 35.40% | train_error: 1.02 | train_acc: 0.951 | val_error: 2.61 | val_acc: 0.874 
+
    +
    +
      [=============>--------------------------] 35.50% | train_error: 1.02 | train_acc: 0.951 | val_error: 2.61 | val_acc: 0.874 
+
    +
    +
      [=============>--------------------------] 35.60% | train_error: 1.02 | train_acc: 0.951 | val_error: 2.61 | val_acc: 0.874 
+
    +
    +
      [=============>--------------------------] 35.70% | train_error: 1.02 | train_acc: 0.951 | val_error: 2.61 | val_acc: 0.874 
+
    +
    +
      [=============>--------------------------] 35.80% | train_error: 1.02 | train_acc: 0.951 | val_error: 2.61 | val_acc: 0.874 
+
    +
    +
      [=============>--------------------------] 35.90% | train_error: 0.876 | train_acc: 0.958 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [=============>--------------------------] 36.00% | train_error: 0.876 | train_acc: 0.958 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [=============>--------------------------] 36.10% | train_error: 0.778 | train_acc: 0.962 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [=============>--------------------------] 36.20% | train_error: 0.778 | train_acc: 0.962 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [=============>--------------------------] 36.30% | train_error: 0.778 | train_acc: 0.962 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [=============>--------------------------] 36.40% | train_error: 0.730 | train_acc: 0.965 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [=============>--------------------------] 36.50% | train_error: 0.730 | train_acc: 0.965 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [=============>--------------------------] 36.60% | train_error: 0.730 | train_acc: 0.965 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [=============>--------------------------] 36.70% | train_error: 0.730 | train_acc: 0.965 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [=============>--------------------------] 36.80% | train_error: 0.730 | train_acc: 0.965 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [=============>--------------------------] 36.90% | train_error: 0.730 | train_acc: 0.965 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [=============>--------------------------] 37.00% | train_error: 0.681 | train_acc: 0.967 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [=============>--------------------------] 37.10% | train_error: 0.681 | train_acc: 0.967 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [=============>--------------------------] 37.20% | train_error: 0.681 | train_acc: 0.967 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [=============>--------------------------] 37.30% | train_error: 0.632 | train_acc: 0.969 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [=============>--------------------------] 37.40% | train_error: 0.632 | train_acc: 0.969 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 37.50% | train_error: 0.632 | train_acc: 0.969 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 37.60% | train_error: 0.632 | train_acc: 0.969 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 37.70% | train_error: 0.632 | train_acc: 0.969 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 37.80% | train_error: 0.632 | train_acc: 0.969 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 37.90% | train_error: 0.632 | train_acc: 0.969 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 38.00% | train_error: 0.535 | train_acc: 0.974 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 38.10% | train_error: 0.535 | train_acc: 0.974 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 38.20% | train_error: 0.486 | train_acc: 0.977 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 38.30% | train_error: 0.341 | train_acc: 0.984 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 38.40% | train_error: 0.486 | train_acc: 0.977 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 38.50% | train_error: 0.341 | train_acc: 0.984 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 38.60% | train_error: 0.486 | train_acc: 0.977 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 38.70% | train_error: 0.341 | train_acc: 0.984 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 38.80% | train_error: 0.486 | train_acc: 0.977 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 38.90% | train_error: 0.341 | train_acc: 0.984 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 39.00% | train_error: 0.486 | train_acc: 0.977 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 39.10% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 39.20% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 39.30% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 39.40% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 39.50% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 39.60% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 39.70% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 39.80% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [==============>-------------------------] 39.90% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [===============>------------------------] 40.00% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [===============>------------------------] 40.10% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [===============>------------------------] 40.20% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [===============>------------------------] 40.30% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [===============>------------------------] 40.40% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [===============>------------------------] 40.50% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [===============>------------------------] 40.60% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [===============>------------------------] 40.70% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [===============>------------------------] 40.80% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [===============>------------------------] 40.90% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [===============>------------------------] 41.00% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [===============>------------------------] 41.10% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [===============>------------------------] 41.20% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [===============>------------------------] 41.30% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [===============>------------------------] 41.40% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [===============>------------------------] 41.50% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [===============>------------------------] 41.60% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [===============>------------------------] 41.70% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [===============>------------------------] 41.80% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [===============>------------------------] 41.90% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [===============>------------------------] 42.00% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [===============>------------------------] 42.10% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [===============>------------------------] 42.20% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [===============>------------------------] 42.30% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [===============>------------------------] 42.40% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [================>-----------------------] 42.50% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [================>-----------------------] 42.60% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [================>-----------------------] 42.70% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [================>-----------------------] 42.80% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [================>-----------------------] 42.90% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [================>-----------------------] 43.00% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [================>-----------------------] 43.10% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [================>-----------------------] 43.20% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [================>-----------------------] 43.30% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [================>-----------------------] 43.40% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [================>-----------------------] 43.50% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [================>-----------------------] 43.60% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [================>-----------------------] 43.70% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [================>-----------------------] 43.80% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [================>-----------------------] 43.90% | train_error: 0.243 | train_acc: 0.988 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [================>-----------------------] 44.00% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [================>-----------------------] 44.10% | train_error: 0.243 | train_acc: 0.988 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [================>-----------------------] 44.20% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.46 | val_acc: 0.881 
+
    +
    +
      [================>-----------------------] 44.30% | train_error: 0.243 | train_acc: 0.988 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [================>-----------------------] 44.40% | train_error: 0.292 | train_acc: 0.986 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [================>-----------------------] 44.50% | train_error: 0.243 | train_acc: 0.988 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [================>-----------------------] 44.60% | train_error: 0.243 | train_acc: 0.988 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [================>-----------------------] 44.70% | train_error: 0.243 | train_acc: 0.988 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [================>-----------------------] 44.80% | train_error: 0.243 | train_acc: 0.988 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [================>-----------------------] 44.90% | train_error: 0.243 | train_acc: 0.988 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [=================>----------------------] 45.00% | train_error: 0.243 | train_acc: 0.988 | val_error: 2.32 | val_acc: 0.888 
+
    +
    +
      [=================>----------------------] 45.10% | train_error: 0.243 | train_acc: 0.988 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [=================>----------------------] 45.20% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [=================>----------------------] 45.30% | train_error: 0.243 | train_acc: 0.988 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [=================>----------------------] 45.40% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [=================>----------------------] 45.50% | train_error: 0.243 | train_acc: 0.988 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [=================>----------------------] 45.60% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [=================>----------------------] 45.70% | train_error: 0.243 | train_acc: 0.988 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [=================>----------------------] 45.80% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [=================>----------------------] 45.90% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [=================>----------------------] 46.00% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [=================>----------------------] 46.10% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=================>----------------------] 46.20% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [=================>----------------------] 46.30% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=================>----------------------] 46.40% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [=================>----------------------] 46.50% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=================>----------------------] 46.60% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [=================>----------------------] 46.70% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=================>----------------------] 46.80% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [=================>----------------------] 46.90% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=================>----------------------] 47.00% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [=================>----------------------] 47.10% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=================>----------------------] 47.20% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [=================>----------------------] 47.30% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=================>----------------------] 47.40% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [==================>---------------------] 47.50% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [==================>---------------------] 47.60% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [==================>---------------------] 47.70% | train_error: 0.195 | train_acc: 0.991 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [==================>---------------------] 47.80% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [==================>---------------------] 47.90% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [==================>---------------------] 48.00% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [==================>---------------------] 48.10% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [==================>---------------------] 48.20% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [==================>---------------------] 48.30% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [==================>---------------------] 48.40% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [==================>---------------------] 48.50% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [==================>---------------------] 48.60% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [==================>---------------------] 48.70% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [==================>---------------------] 48.80% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [==================>---------------------] 48.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [==================>---------------------] 49.00% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [==================>---------------------] 49.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [==================>---------------------] 49.20% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [==================>---------------------] 49.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [==================>---------------------] 49.40% | train_error: 0.146 | train_acc: 0.993 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [==================>---------------------] 49.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [==================>---------------------] 49.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [==================>---------------------] 49.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [==================>---------------------] 49.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [==================>---------------------] 49.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [===================>--------------------] 50.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [===================>--------------------] 50.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [===================>--------------------] 50.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [===================>--------------------] 50.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [===================>--------------------] 50.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [===================>--------------------] 50.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [===================>--------------------] 50.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [===================>--------------------] 50.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [===================>--------------------] 50.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [===================>--------------------] 50.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [===================>--------------------] 51.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [===================>--------------------] 51.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [===================>--------------------] 51.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [===================>--------------------] 51.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [===================>--------------------] 51.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [===================>--------------------] 51.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [===================>--------------------] 51.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [===================>--------------------] 51.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [===================>--------------------] 51.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [===================>--------------------] 51.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [===================>--------------------] 52.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.17 | val_acc: 0.895 
+
    +
    +
      [===================>--------------------] 52.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [===================>--------------------] 52.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [===================>--------------------] 52.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [===================>--------------------] 52.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [====================>-------------------] 52.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [====================>-------------------] 52.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [====================>-------------------] 52.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [====================>-------------------] 52.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [====================>-------------------] 52.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [====================>-------------------] 53.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [====================>-------------------] 53.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [====================>-------------------] 53.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [====================>-------------------] 53.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [====================>-------------------] 53.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [====================>-------------------] 53.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [====================>-------------------] 53.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [====================>-------------------] 53.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [====================>-------------------] 53.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [====================>-------------------] 53.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [====================>-------------------] 54.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [====================>-------------------] 54.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [====================>-------------------] 54.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [====================>-------------------] 54.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [====================>-------------------] 54.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [====================>-------------------] 54.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [====================>-------------------] 54.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [====================>-------------------] 54.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [====================>-------------------] 54.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [====================>-------------------] 54.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=====================>------------------] 55.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=====================>------------------] 55.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=====================>------------------] 55.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=====================>------------------] 55.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=====================>------------------] 55.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=====================>------------------] 55.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=====================>------------------] 55.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=====================>------------------] 55.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=====================>------------------] 55.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=====================>------------------] 55.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=====================>------------------] 56.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=====================>------------------] 56.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=====================>------------------] 56.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=====================>------------------] 56.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=====================>------------------] 56.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=====================>------------------] 56.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=====================>------------------] 56.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=====================>------------------] 56.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=====================>------------------] 56.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=====================>------------------] 56.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=====================>------------------] 57.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=====================>------------------] 57.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=====================>------------------] 57.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 2.03 | val_acc: 0.902 
+
    +
    +
      [=====================>------------------] 57.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=====================>------------------] 57.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 57.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 57.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 57.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 57.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 57.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 58.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 58.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 58.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 58.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 58.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 58.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 58.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 58.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 58.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 58.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 59.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 59.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 59.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 59.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 59.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 59.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 59.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 59.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 59.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [======================>-----------------] 59.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 60.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 60.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 60.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 60.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 60.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 60.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 60.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 60.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 60.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 60.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 61.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 61.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 61.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 61.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 61.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 61.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 61.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 61.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 61.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 61.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 62.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 62.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 62.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 62.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=======================>----------------] 62.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 62.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 62.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 62.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 62.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 62.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 63.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 63.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 63.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 63.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 63.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 63.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 63.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 63.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 63.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 63.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [========================>---------------] 64.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 64.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [========================>---------------] 64.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 64.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [========================>---------------] 64.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 64.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [========================>---------------] 64.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 64.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [========================>---------------] 64.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [========================>---------------] 64.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=========================>--------------] 65.00% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=========================>--------------] 65.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=========================>--------------] 65.20% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=========================>--------------] 65.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=========================>--------------] 65.40% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=========================>--------------] 65.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=========================>--------------] 65.60% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=========================>--------------] 65.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=========================>--------------] 65.80% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=========================>--------------] 65.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=========================>--------------] 66.00% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=========================>--------------] 66.10% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=========================>--------------] 66.20% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=========================>--------------] 66.30% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=========================>--------------] 66.40% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=========================>--------------] 66.50% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=========================>--------------] 66.60% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=========================>--------------] 66.70% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=========================>--------------] 66.80% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=========================>--------------] 66.90% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=========================>--------------] 67.00% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=========================>--------------] 67.10% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=========================>--------------] 67.20% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [=========================>--------------] 67.30% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=========================>--------------] 67.40% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [==========================>-------------] 67.50% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==========================>-------------] 67.60% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [==========================>-------------] 67.70% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==========================>-------------] 67.80% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [==========================>-------------] 67.90% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==========================>-------------] 68.00% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [==========================>-------------] 68.10% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==========================>-------------] 68.20% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [==========================>-------------] 68.30% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==========================>-------------] 68.40% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [==========================>-------------] 68.50% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==========================>-------------] 68.60% | train_error: 0.146 | train_acc: 0.993 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [==========================>-------------] 68.70% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==========================>-------------] 68.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [==========================>-------------] 68.90% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==========================>-------------] 69.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [==========================>-------------] 69.10% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==========================>-------------] 69.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [==========================>-------------] 69.30% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==========================>-------------] 69.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [==========================>-------------] 69.50% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==========================>-------------] 69.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [==========================>-------------] 69.70% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==========================>-------------] 69.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.88 | val_acc: 0.909 
+
    +
    +
      [==========================>-------------] 69.90% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 70.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 70.10% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 70.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 70.30% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 70.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 70.50% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 70.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 70.70% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 70.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 70.90% | train_error: 0.0486 | train_acc: 0.998 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 71.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 71.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 71.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 71.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 71.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 71.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 71.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 71.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 71.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 71.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 72.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 72.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 72.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 72.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===========================>------------] 72.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 72.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 72.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 72.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 72.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 72.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 73.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 73.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 73.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 73.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 73.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 73.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 73.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 73.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 73.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 73.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 74.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 74.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 74.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 74.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 74.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 74.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 74.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 74.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 74.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [============================>-----------] 74.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 75.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 75.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 75.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 75.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 75.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 75.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 75.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 75.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 75.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 75.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 76.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 76.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 76.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 76.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 76.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 76.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 76.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 76.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 76.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 76.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 77.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 77.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [=============================>----------] 77.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=============================>----------] 77.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [=============================>----------] 77.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==============================>---------] 77.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==============================>---------] 77.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==============================>---------] 77.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==============================>---------] 77.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==============================>---------] 77.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==============================>---------] 78.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==============================>---------] 78.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==============================>---------] 78.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==============================>---------] 78.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==============================>---------] 78.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==============================>---------] 78.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==============================>---------] 78.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==============================>---------] 78.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==============================>---------] 78.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==============================>---------] 78.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==============================>---------] 79.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==============================>---------] 79.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==============================>---------] 79.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==============================>---------] 79.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==============================>---------] 79.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==============================>---------] 79.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==============================>---------] 79.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==============================>---------] 79.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==============================>---------] 79.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==============================>---------] 79.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [===============================>--------] 80.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===============================>--------] 80.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [===============================>--------] 80.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===============================>--------] 80.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [===============================>--------] 80.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===============================>--------] 80.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [===============================>--------] 80.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===============================>--------] 80.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [===============================>--------] 80.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===============================>--------] 80.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [===============================>--------] 81.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===============================>--------] 81.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [===============================>--------] 81.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===============================>--------] 81.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [===============================>--------] 81.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===============================>--------] 81.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [===============================>--------] 81.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===============================>--------] 81.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [===============================>--------] 81.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===============================>--------] 81.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [===============================>--------] 82.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===============================>--------] 82.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [===============================>--------] 82.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===============================>--------] 82.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [===============================>--------] 82.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [================================>-------] 82.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [================================>-------] 82.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [================================>-------] 82.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [================================>-------] 82.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [================================>-------] 82.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [================================>-------] 83.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [================================>-------] 83.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [================================>-------] 83.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [================================>-------] 83.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [================================>-------] 83.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [================================>-------] 83.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [================================>-------] 83.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [================================>-------] 83.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [================================>-------] 83.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [================================>-------] 83.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [================================>-------] 84.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [================================>-------] 84.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [================================>-------] 84.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [================================>-------] 84.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [================================>-------] 84.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [================================>-------] 84.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [================================>-------] 84.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [================================>-------] 84.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [================================>-------] 84.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [================================>-------] 84.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [=================================>------] 85.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=================================>------] 85.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [=================================>------] 85.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=================================>------] 85.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [=================================>------] 85.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=================================>------] 85.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [=================================>------] 85.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=================================>------] 85.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [=================================>------] 85.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=================================>------] 85.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [=================================>------] 86.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=================================>------] 86.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [=================================>------] 86.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=================================>------] 86.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [=================================>------] 86.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=================================>------] 86.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [=================================>------] 86.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=================================>------] 86.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [=================================>------] 86.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=================================>------] 86.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [=================================>------] 87.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=================================>------] 87.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [=================================>------] 87.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=================================>------] 87.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [=================================>------] 87.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==================================>-----] 87.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==================================>-----] 87.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==================================>-----] 87.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==================================>-----] 87.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==================================>-----] 87.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==================================>-----] 88.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==================================>-----] 88.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==================================>-----] 88.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==================================>-----] 88.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==================================>-----] 88.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==================================>-----] 88.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==================================>-----] 88.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==================================>-----] 88.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==================================>-----] 88.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==================================>-----] 88.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==================================>-----] 89.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==================================>-----] 89.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==================================>-----] 89.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==================================>-----] 89.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==================================>-----] 89.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==================================>-----] 89.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==================================>-----] 89.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==================================>-----] 89.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [==================================>-----] 89.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [==================================>-----] 89.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.59 | val_acc: 0.923 
+
    +
    +
      [===================================>----] 90.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 90.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 90.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 90.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 90.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 90.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 90.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 90.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 90.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 90.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 91.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 91.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 91.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 91.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 91.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 91.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 91.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 91.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 91.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 91.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 92.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 92.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 92.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 92.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [===================================>----] 92.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 92.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 92.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 92.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 92.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 92.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 93.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 93.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 93.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 93.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 93.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 93.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 93.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 93.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 93.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 93.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 94.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 94.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 94.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 94.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 94.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 94.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 94.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 94.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 94.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [====================================>---] 94.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 95.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 95.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 95.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 95.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 95.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 95.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 95.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 95.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 95.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 95.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 96.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 96.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 96.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 96.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 96.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 96.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 96.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 96.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 96.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 96.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 97.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 97.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 97.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 97.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [=====================================>--] 97.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 97.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 97.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 97.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 97.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 97.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 98.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 98.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 98.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 98.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 98.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 98.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 98.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 98.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 98.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 98.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 99.00% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 99.10% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 99.20% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 99.30% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 99.40% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 99.50% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 99.60% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 99.70% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 99.80% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
      [======================================>-] 99.90% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
                                                                                                                                    
+
    +
    +
      [=======================================>] 100.0% | train_error: 0.0973 | train_acc: 0.995 | val_error: 1.74 | val_acc: 0.916 
+
    +
    +
    +
    +
    +
    +

    Multiclass classification

    +

    Finally, we will demonstrate the use case of multiclass classification +using our FFNN with the famous MNIST dataset, which contain images of +digits between the range of 0 to 9.

    +
    +
    +
    from sklearn.datasets import load_digits
+
+def onehot(target: np.ndarray):
+    onehot = np.zeros((target.size, target.max() + 1))
+    onehot[np.arange(target.size), target] = 1
+    return onehot
+
+digits = load_digits()
+
+X = digits.data
+target = digits.target
+target = onehot(target)
+
+input_nodes = 64
+hidden_nodes1 = 100
+hidden_nodes2 = 30
+output_nodes = 10
+
+dims = (input_nodes, hidden_nodes1, hidden_nodes2, output_nodes)
+
+multiclass = FFNN(dims, hidden_func=LRELU, output_func=softmax, cost_func=CostCrossEntropy)
+
+multiclass.reset_weights() # reset weights such that previous runs or reruns don't affect the weights
+
+scheduler = Adam(eta=1e-4, rho=0.9, rho2=0.999)
+scores = multiclass.fit(X, target, scheduler, epochs=1000)
+
    +
    +
    +
    +
    Adam: Eta=0.0001, Lambda=0
+
    +
    +
      [----------------------------------------] 0.000% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [----------------------------------------] 0.1000% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [----------------------------------------] 0.2000% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [----------------------------------------] 0.3000% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [----------------------------------------] 0.4000% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [----------------------------------------] 0.5000% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [----------------------------------------] 0.6000% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 0.7000% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 0.8000% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 0.9000% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 1.000% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 1.100% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 1.200% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 1.300% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 1.400% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 1.500% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 1.600% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 1.700% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 1.800% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 1.900% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 2.000% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 2.100% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 2.200% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 2.300% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [----------------------------------------] 2.400% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 2.500% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 2.600% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 2.700% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 2.800% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 2.900% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 3.000% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 3.100% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 3.200% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 3.300% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 3.400% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 3.500% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 3.600% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 3.700% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 3.800% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 3.900% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 4.000% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 4.100% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 4.200% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 4.300% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [>---------------------------------------] 4.400% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [>---------------------------------------] 4.500% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [>---------------------------------------] 4.600% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [>---------------------------------------] 4.700% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [>---------------------------------------] 4.800% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [>---------------------------------------] 4.900% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [=>--------------------------------------] 5.000% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [=>--------------------------------------] 5.100% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [=>--------------------------------------] 5.200% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [=>--------------------------------------] 5.300% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [=>--------------------------------------] 5.400% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [=>--------------------------------------] 5.500% | train_error: 1.84 | train_acc: 0.823 
+
    +
    +
      [=>--------------------------------------] 5.600% | train_error: 1.84 | train_acc: 0.823 
+
    +
    +
      [=>--------------------------------------] 5.700% | train_error: 1.84 | train_acc: 0.822 
+
    +
    +
      [=>--------------------------------------] 5.800% | train_error: 1.84 | train_acc: 0.822 
+
    +
    +
      [=>--------------------------------------] 5.900% | train_error: 1.84 | train_acc: 0.822 
+
    +
    +
      [=>--------------------------------------] 6.000% | train_error: 1.84 | train_acc: 0.823 
+
    +
    +
      [=>--------------------------------------] 6.100% | train_error: 1.84 | train_acc: 0.823 
+
    +
    +
      [=>--------------------------------------] 6.200% | train_error: 1.84 | train_acc: 0.823 
+
    +
    +
      [=>--------------------------------------] 6.300% | train_error: 1.84 | train_acc: 0.823 
+
    +
    +
      [=>--------------------------------------] 6.400% | train_error: 1.84 | train_acc: 0.823 
+
    +
    +
      [=>--------------------------------------] 6.500% | train_error: 1.84 | train_acc: 0.823 
+
    +
    +
      [=>--------------------------------------] 6.600% | train_error: 1.84 | train_acc: 0.823 
+
    +
    +
      [=>--------------------------------------] 6.700% | train_error: 1.84 | train_acc: 0.823 
+
    +
    +
      [=>--------------------------------------] 6.800% | train_error: 1.84 | train_acc: 0.822 
+
    +
    +
      [=>--------------------------------------] 6.900% | train_error: 1.85 | train_acc: 0.822 
+
    +
    +
      [=>--------------------------------------] 7.000% | train_error: 1.85 | train_acc: 0.822 
+
    +
    +
      [=>--------------------------------------] 7.100% | train_error: 1.85 | train_acc: 0.821 
+
    +
    +
      [=>--------------------------------------] 7.200% | train_error: 1.85 | train_acc: 0.821 
+
    +
    +
      [=>--------------------------------------] 7.300% | train_error: 1.85 | train_acc: 0.821 
+
    +
    +
      [=>--------------------------------------] 7.400% | train_error: 1.85 | train_acc: 0.821 
+
    +
    +
      [==>-------------------------------------] 7.500% | train_error: 1.86 | train_acc: 0.821 
+
    +
    +
      [==>-------------------------------------] 7.600% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [==>-------------------------------------] 7.700% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [==>-------------------------------------] 7.800% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [==>-------------------------------------] 7.900% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [==>-------------------------------------] 8.000% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [==>-------------------------------------] 8.100% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [==>-------------------------------------] 8.200% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [==>-------------------------------------] 8.300% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [==>-------------------------------------] 8.400% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [==>-------------------------------------] 8.500% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [==>-------------------------------------] 8.600% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [==>-------------------------------------] 8.700% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [==>-------------------------------------] 8.800% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [==>-------------------------------------] 8.900% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [==>-------------------------------------] 9.000% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [==>-------------------------------------] 9.100% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [==>-------------------------------------] 9.200% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [==>-------------------------------------] 9.300% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [==>-------------------------------------] 9.400% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [==>-------------------------------------] 9.500% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [==>-------------------------------------] 9.600% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [==>-------------------------------------] 9.700% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [==>-------------------------------------] 9.800% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [==>-------------------------------------] 9.900% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [===>------------------------------------] 10.00% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [===>------------------------------------] 10.10% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [===>------------------------------------] 10.20% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [===>------------------------------------] 10.30% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [===>------------------------------------] 10.40% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [===>------------------------------------] 10.50% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [===>------------------------------------] 10.60% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [===>------------------------------------] 10.70% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [===>------------------------------------] 10.80% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [===>------------------------------------] 10.90% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [===>------------------------------------] 11.00% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [===>------------------------------------] 11.10% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [===>------------------------------------] 11.20% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [===>------------------------------------] 11.30% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [===>------------------------------------] 11.40% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [===>------------------------------------] 11.50% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [===>------------------------------------] 11.60% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [===>------------------------------------] 11.70% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [===>------------------------------------] 11.80% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [===>------------------------------------] 11.90% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [===>------------------------------------] 12.00% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [===>------------------------------------] 12.10% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [===>------------------------------------] 12.20% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [===>------------------------------------] 12.30% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [===>------------------------------------] 12.40% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [====>-----------------------------------] 12.50% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [====>-----------------------------------] 12.60% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [====>-----------------------------------] 12.70% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [====>-----------------------------------] 12.80% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [====>-----------------------------------] 12.90% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [====>-----------------------------------] 13.00% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [====>-----------------------------------] 13.10% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [====>-----------------------------------] 13.20% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [====>-----------------------------------] 13.30% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [====>-----------------------------------] 13.40% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [====>-----------------------------------] 13.50% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [====>-----------------------------------] 13.60% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [====>-----------------------------------] 13.70% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [====>-----------------------------------] 13.80% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [====>-----------------------------------] 13.90% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [====>-----------------------------------] 14.00% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [====>-----------------------------------] 14.10% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [====>-----------------------------------] 14.20% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [====>-----------------------------------] 14.30% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [====>-----------------------------------] 14.40% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [====>-----------------------------------] 14.50% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [====>-----------------------------------] 14.60% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [====>-----------------------------------] 14.70% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [====>-----------------------------------] 14.80% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [====>-----------------------------------] 14.90% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [=====>----------------------------------] 15.00% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [=====>----------------------------------] 15.10% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [=====>----------------------------------] 15.20% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [=====>----------------------------------] 15.30% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [=====>----------------------------------] 15.40% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [=====>----------------------------------] 15.50% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [=====>----------------------------------] 15.60% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [=====>----------------------------------] 15.70% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [=====>----------------------------------] 15.80% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [=====>----------------------------------] 15.90% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [=====>----------------------------------] 16.00% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [=====>----------------------------------] 16.10% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [=====>----------------------------------] 16.20% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [=====>----------------------------------] 16.30% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [=====>----------------------------------] 16.40% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [=====>----------------------------------] 16.50% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=====>----------------------------------] 16.60% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=====>----------------------------------] 16.70% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=====>----------------------------------] 16.80% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=====>----------------------------------] 16.90% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=====>----------------------------------] 17.00% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=====>----------------------------------] 17.10% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=====>----------------------------------] 17.20% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=====>----------------------------------] 17.30% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=====>----------------------------------] 17.40% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 17.50% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 17.60% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 17.70% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 17.80% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 17.90% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 18.00% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 18.10% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 18.20% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [======>---------------------------------] 18.30% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [======>---------------------------------] 18.40% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 18.50% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [======>---------------------------------] 18.60% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 18.70% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 18.80% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 18.90% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 19.00% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 19.10% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 19.20% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 19.30% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 19.40% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 19.50% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 19.60% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 19.70% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [======>---------------------------------] 19.80% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [======>---------------------------------] 19.90% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 20.00% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 20.10% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 20.20% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 20.30% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 20.40% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [=======>--------------------------------] 20.50% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 20.60% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 20.70% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 20.80% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 20.90% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 21.00% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 21.10% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 21.20% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 21.30% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 21.40% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 21.50% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [=======>--------------------------------] 21.60% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [=======>--------------------------------] 21.70% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [=======>--------------------------------] 21.80% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [=======>--------------------------------] 21.90% | train_error: 1.88 | train_acc: 0.818 
+
    +
    +
      [=======>--------------------------------] 22.00% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 22.10% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 22.20% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 22.30% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [=======>--------------------------------] 22.40% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [========>-------------------------------] 22.50% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [========>-------------------------------] 22.60% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [========>-------------------------------] 22.70% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [========>-------------------------------] 22.80% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [========>-------------------------------] 22.90% | train_error: 1.88 | train_acc: 0.819 
+
    +
    +
      [========>-------------------------------] 23.00% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [========>-------------------------------] 23.10% | train_error: 1.87 | train_acc: 0.819 
+
    +
    +
      [========>-------------------------------] 23.20% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [========>-------------------------------] 23.30% | train_error: 1.87 | train_acc: 0.820 
+
    +
    +
      [========>-------------------------------] 23.40% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [========>-------------------------------] 23.50% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [========>-------------------------------] 23.60% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [========>-------------------------------] 23.70% | train_error: 1.86 | train_acc: 0.821 
+
    +
    +
      [========>-------------------------------] 23.80% | train_error: 1.86 | train_acc: 0.821 
+
    +
    +
      [========>-------------------------------] 23.90% | train_error: 1.86 | train_acc: 0.821 
+
    +
    +
      [========>-------------------------------] 24.00% | train_error: 1.86 | train_acc: 0.821 
+
    +
    +
      [========>-------------------------------] 24.10% | train_error: 1.86 | train_acc: 0.821 
+
    +
    +
      [========>-------------------------------] 24.20% | train_error: 1.86 | train_acc: 0.820 
+
    +
    +
      [========>-------------------------------] 24.30% | train_error: 1.86 | train_acc: 0.821 
+
    +
    +
      [========>-------------------------------] 24.40% | train_error: 1.86 | train_acc: 0.821 
+
    +
    +
      [========>-------------------------------] 24.50% | train_error: 1.85 | train_acc: 0.821 
+
    +
    +
      [========>-------------------------------] 24.60% | train_error: 1.85 | train_acc: 0.821 
+
    +
    +
      [========>-------------------------------] 24.70% | train_error: 1.85 | train_acc: 0.822 
+
    +
    +
      [========>-------------------------------] 24.80% | train_error: 1.85 | train_acc: 0.821 
+
    +
    +
      [========>-------------------------------] 24.90% | train_error: 1.85 | train_acc: 0.822 
+
    +
    +
      [=========>------------------------------] 25.00% | train_error: 1.85 | train_acc: 0.822 
+
    +
    +
      [=========>------------------------------] 25.10% | train_error: 1.84 | train_acc: 0.822 
+
    +
    +
      [=========>------------------------------] 25.20% | train_error: 1.84 | train_acc: 0.822 
+
    +
    +
      [=========>------------------------------] 25.30% | train_error: 1.84 | train_acc: 0.822 
+
    +
    +
      [=========>------------------------------] 25.40% | train_error: 1.84 | train_acc: 0.822 
+
    +
    +
      [=========>------------------------------] 25.50% | train_error: 1.84 | train_acc: 0.822 
+
    +
    +
      [=========>------------------------------] 25.60% | train_error: 1.84 | train_acc: 0.823 
+
    +
    +
      [=========>------------------------------] 25.70% | train_error: 1.84 | train_acc: 0.823 
+
    +
    +
      [=========>------------------------------] 25.80% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [=========>------------------------------] 25.90% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [=========>------------------------------] 26.00% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [=========>------------------------------] 26.10% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [=========>------------------------------] 26.20% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [=========>------------------------------] 26.30% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [=========>------------------------------] 26.40% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [=========>------------------------------] 26.50% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [=========>------------------------------] 26.60% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [=========>------------------------------] 26.70% | train_error: 1.83 | train_acc: 0.823 
+
    +
    +
      [=========>------------------------------] 26.80% | train_error: 1.83 | train_acc: 0.824 
+
    +
    +
      [=========>------------------------------] 26.90% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [=========>------------------------------] 27.00% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [=========>------------------------------] 27.10% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [=========>------------------------------] 27.20% | train_error: 1.82 | train_acc: 0.824 
+
    +
    +
      [=========>------------------------------] 27.30% | train_error: 1.82 | train_acc: 0.825 
+
    +
    +
      [=========>------------------------------] 27.40% | train_error: 1.82 | train_acc: 0.825 
+
    +
    +
      [==========>-----------------------------] 27.50% | train_error: 1.82 | train_acc: 0.825 
+
    +
    +
      [==========>-----------------------------] 27.60% | train_error: 1.82 | train_acc: 0.825 
+
    +
    +
      [==========>-----------------------------] 27.70% | train_error: 1.81 | train_acc: 0.825 
+
    +
    +
      [==========>-----------------------------] 27.80% | train_error: 1.81 | train_acc: 0.825 
+
    +
    +
      [==========>-----------------------------] 27.90% | train_error: 1.81 | train_acc: 0.825 
+
    +
    +
      [==========>-----------------------------] 28.00% | train_error: 1.81 | train_acc: 0.825 
+
    +
    +
      [==========>-----------------------------] 28.10% | train_error: 1.81 | train_acc: 0.825 
+
    +
    +
      [==========>-----------------------------] 28.20% | train_error: 1.81 | train_acc: 0.825 
+
    +
    +
      [==========>-----------------------------] 28.30% | train_error: 1.81 | train_acc: 0.825 
+
    +
    +
      [==========>-----------------------------] 28.40% | train_error: 1.80 | train_acc: 0.826 
+
    +
    +
      [==========>-----------------------------] 28.50% | train_error: 1.80 | train_acc: 0.826 
+
    +
    +
      [==========>-----------------------------] 28.60% | train_error: 1.80 | train_acc: 0.826 
+
    +
    +
      [==========>-----------------------------] 28.70% | train_error: 1.80 | train_acc: 0.826 
+
    +
    +
      [==========>-----------------------------] 28.80% | train_error: 1.80 | train_acc: 0.826 
+
    +
    +
      [==========>-----------------------------] 28.90% | train_error: 1.80 | train_acc: 0.826 
+
    +
    +
      [==========>-----------------------------] 29.00% | train_error: 1.80 | train_acc: 0.826 
+
    +
    +
      [==========>-----------------------------] 29.10% | train_error: 1.80 | train_acc: 0.826 
+
    +
    +
      [==========>-----------------------------] 29.20% | train_error: 1.80 | train_acc: 0.827 
+
    +
    +
      [==========>-----------------------------] 29.30% | train_error: 1.80 | train_acc: 0.827 
+
    +
    +
      [==========>-----------------------------] 29.40% | train_error: 1.80 | train_acc: 0.826 
+
    +
    +
      [==========>-----------------------------] 29.50% | train_error: 1.80 | train_acc: 0.827 
+
    +
    +
      [==========>-----------------------------] 29.60% | train_error: 1.80 | train_acc: 0.827 
+
    +
    +
      [==========>-----------------------------] 29.70% | train_error: 1.80 | train_acc: 0.826 
+
    +
    +
      [==========>-----------------------------] 29.80% | train_error: 1.79 | train_acc: 0.827 
+
    +
    +
      [==========>-----------------------------] 29.90% | train_error: 1.79 | train_acc: 0.827 
+
    +
    +
      [===========>----------------------------] 30.00% | train_error: 1.79 | train_acc: 0.827 
+
    +
    +
      [===========>----------------------------] 30.10% | train_error: 1.79 | train_acc: 0.828 
+
    +
    +
      [===========>----------------------------] 30.20% | train_error: 1.79 | train_acc: 0.827 
+
    +
    +
      [===========>----------------------------] 30.30% | train_error: 1.79 | train_acc: 0.827 
+
    +
    +
      [===========>----------------------------] 30.40% | train_error: 1.79 | train_acc: 0.827 
+
    +
    +
      [===========>----------------------------] 30.50% | train_error: 1.79 | train_acc: 0.828 
+
    +
    +
      [===========>----------------------------] 30.60% | train_error: 1.78 | train_acc: 0.828 
+
    +
    +
      [===========>----------------------------] 30.70% | train_error: 1.78 | train_acc: 0.828 
+
    +
    +
      [===========>----------------------------] 30.80% | train_error: 1.78 | train_acc: 0.828 
+
    +
    +
      [===========>----------------------------] 30.90% | train_error: 1.78 | train_acc: 0.828 
+
    +
    +
      [===========>----------------------------] 31.00% | train_error: 1.78 | train_acc: 0.828 
+
    +
    +
      [===========>----------------------------] 31.10% | train_error: 1.78 | train_acc: 0.828 
+
    +
    +
      [===========>----------------------------] 31.20% | train_error: 1.78 | train_acc: 0.828 
+
    +
    +
      [===========>----------------------------] 31.30% | train_error: 1.78 | train_acc: 0.828 
+
    +
    +
      [===========>----------------------------] 31.40% | train_error: 1.78 | train_acc: 0.829 
+
    +
    +
      [===========>----------------------------] 31.50% | train_error: 1.78 | train_acc: 0.829 
+
    +
    +
      [===========>----------------------------] 31.60% | train_error: 1.77 | train_acc: 0.829 
+
    +
    +
      [===========>----------------------------] 31.70% | train_error: 1.78 | train_acc: 0.829 
+
    +
    +
      [===========>----------------------------] 31.80% | train_error: 1.78 | train_acc: 0.829 
+
    +
    +
      [===========>----------------------------] 31.90% | train_error: 1.77 | train_acc: 0.829 
+
    +
    +
      [===========>----------------------------] 32.00% | train_error: 1.77 | train_acc: 0.829 
+
    +
    +
      [===========>----------------------------] 32.10% | train_error: 1.76 | train_acc: 0.830 
+
    +
    +
      [===========>----------------------------] 32.20% | train_error: 1.77 | train_acc: 0.829 
+
    +
    +
      [===========>----------------------------] 32.30% | train_error: 1.76 | train_acc: 0.830 
+
    +
    +
      [===========>----------------------------] 32.40% | train_error: 1.76 | train_acc: 0.830 
+
    +
    +
      [============>---------------------------] 32.50% | train_error: 1.76 | train_acc: 0.830 
+
    +
    +
      [============>---------------------------] 32.60% | train_error: 1.76 | train_acc: 0.830 
+
    +
    +
      [============>---------------------------] 32.70% | train_error: 1.76 | train_acc: 0.830 
+
    +
    +
      [============>---------------------------] 32.80% | train_error: 1.76 | train_acc: 0.830 
+
    +
    +
      [============>---------------------------] 32.90% | train_error: 1.76 | train_acc: 0.831 
+
    +
    +
      [============>---------------------------] 33.00% | train_error: 1.75 | train_acc: 0.831 
+
    +
    +
      [============>---------------------------] 33.10% | train_error: 1.75 | train_acc: 0.831 
+
    +
    +
      [============>---------------------------] 33.20% | train_error: 1.75 | train_acc: 0.831 
+
    +
    +
      [============>---------------------------] 33.30% | train_error: 1.75 | train_acc: 0.831 
+
    +
    +
      [============>---------------------------] 33.40% | train_error: 1.75 | train_acc: 0.831 
+
    +
    +
      [============>---------------------------] 33.50% | train_error: 1.75 | train_acc: 0.831 
+
    +
    +
      [============>---------------------------] 33.60% | train_error: 1.75 | train_acc: 0.831 
+
    +
    +
      [============>---------------------------] 33.70% | train_error: 1.75 | train_acc: 0.831 
+
    +
    +
      [============>---------------------------] 33.80% | train_error: 1.75 | train_acc: 0.831 
+
    +
    +
      [============>---------------------------] 33.90% | train_error: 1.75 | train_acc: 0.831 
+
    +
    +
      [============>---------------------------] 34.00% | train_error: 1.75 | train_acc: 0.831 
+
    +
    +
      [============>---------------------------] 34.10% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [============>---------------------------] 34.20% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [============>---------------------------] 34.30% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [============>---------------------------] 34.40% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [============>---------------------------] 34.50% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [============>---------------------------] 34.60% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [============>---------------------------] 34.70% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [============>---------------------------] 34.80% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [============>---------------------------] 34.90% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [=============>--------------------------] 35.00% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [=============>--------------------------] 35.10% | train_error: 1.73 | train_acc: 0.834 
+
    +
    +
      [=============>--------------------------] 35.20% | train_error: 1.73 | train_acc: 0.834 
+
    +
    +
      [=============>--------------------------] 35.30% | train_error: 1.72 | train_acc: 0.834 
+
    +
    +
      [=============>--------------------------] 35.40% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [=============>--------------------------] 35.50% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [=============>--------------------------] 35.60% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [=============>--------------------------] 35.70% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [=============>--------------------------] 35.80% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [=============>--------------------------] 35.90% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [=============>--------------------------] 36.00% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [=============>--------------------------] 36.10% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [=============>--------------------------] 36.20% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [=============>--------------------------] 36.30% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [=============>--------------------------] 36.40% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [=============>--------------------------] 36.50% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [=============>--------------------------] 36.60% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [=============>--------------------------] 36.70% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [=============>--------------------------] 36.80% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [=============>--------------------------] 36.90% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [=============>--------------------------] 37.00% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [=============>--------------------------] 37.10% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [=============>--------------------------] 37.20% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [=============>--------------------------] 37.30% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [=============>--------------------------] 37.40% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [==============>-------------------------] 37.50% | train_error: 1.74 | train_acc: 0.832 
+
    +
    +
      [==============>-------------------------] 37.60% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [==============>-------------------------] 37.70% | train_error: 1.72 | train_acc: 0.834 
+
    +
    +
      [==============>-------------------------] 37.80% | train_error: 1.73 | train_acc: 0.834 
+
    +
    +
      [==============>-------------------------] 37.90% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [==============>-------------------------] 38.00% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [==============>-------------------------] 38.10% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [==============>-------------------------] 38.20% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [==============>-------------------------] 38.30% | train_error: 1.72 | train_acc: 0.834 
+
    +
    +
      [==============>-------------------------] 38.40% | train_error: 1.73 | train_acc: 0.834 
+
    +
    +
      [==============>-------------------------] 38.50% | train_error: 1.73 | train_acc: 0.834 
+
    +
    +
      [==============>-------------------------] 38.60% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [==============>-------------------------] 38.70% | train_error: 1.73 | train_acc: 0.834 
+
    +
    +
      [==============>-------------------------] 38.80% | train_error: 1.72 | train_acc: 0.834 
+
    +
    +
      [==============>-------------------------] 38.90% | train_error: 1.72 | train_acc: 0.834 
+
    +
    +
      [==============>-------------------------] 39.00% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [==============>-------------------------] 39.10% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [==============>-------------------------] 39.20% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [==============>-------------------------] 39.30% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [==============>-------------------------] 39.40% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [==============>-------------------------] 39.50% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [==============>-------------------------] 39.60% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [==============>-------------------------] 39.70% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [==============>-------------------------] 39.80% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [==============>-------------------------] 39.90% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [===============>------------------------] 40.00% | train_error: 1.73 | train_acc: 0.833 
+
    +
    +
      [===============>------------------------] 40.10% | train_error: 1.72 | train_acc: 0.834 
+
    +
    +
      [===============>------------------------] 40.20% | train_error: 1.72 | train_acc: 0.834 
+
    +
    +
      [===============>------------------------] 40.30% | train_error: 1.72 | train_acc: 0.834 
+
    +
    +
      [===============>------------------------] 40.40% | train_error: 1.71 | train_acc: 0.835 
+
    +
    +
      [===============>------------------------] 40.50% | train_error: 1.71 | train_acc: 0.835 
+
    +
    +
      [===============>------------------------] 40.60% | train_error: 1.71 | train_acc: 0.835 
+
    +
    +
      [===============>------------------------] 40.70% | train_error: 1.71 | train_acc: 0.835 
+
    +
    +
      [===============>------------------------] 40.80% | train_error: 1.71 | train_acc: 0.835 
+
    +
    +
      [===============>------------------------] 40.90% | train_error: 1.71 | train_acc: 0.835 
+
    +
    +
      [===============>------------------------] 41.00% | train_error: 1.71 | train_acc: 0.835 
+
    +
    +
      [===============>------------------------] 41.10% | train_error: 1.71 | train_acc: 0.835 
+
    +
    +
      [===============>------------------------] 41.20% | train_error: 1.71 | train_acc: 0.835 
+
    +
    +
      [===============>------------------------] 41.30% | train_error: 1.71 | train_acc: 0.835 
+
    +
    +
      [===============>------------------------] 41.40% | train_error: 1.71 | train_acc: 0.835 
+
    +
    +
      [===============>------------------------] 41.50% | train_error: 1.70 | train_acc: 0.836 
+
    +
    +
      [===============>------------------------] 41.60% | train_error: 1.70 | train_acc: 0.836 
+
    +
    +
      [===============>------------------------] 41.70% | train_error: 1.70 | train_acc: 0.836 
+
    +
    +
      [===============>------------------------] 41.80% | train_error: 1.70 | train_acc: 0.836 
+
    +
    +
      [===============>------------------------] 41.90% | train_error: 1.70 | train_acc: 0.836 
+
    +
    +
      [===============>------------------------] 42.00% | train_error: 1.70 | train_acc: 0.836 
+
    +
    +
      [===============>------------------------] 42.10% | train_error: 1.69 | train_acc: 0.837 
+
    +
    +
      [===============>------------------------] 42.20% | train_error: 1.69 | train_acc: 0.837 
+
    +
    +
      [===============>------------------------] 42.30% | train_error: 1.69 | train_acc: 0.837 
+
    +
    +
      [===============>------------------------] 42.40% | train_error: 1.69 | train_acc: 0.837 
+
    +
    +
      [================>-----------------------] 42.50% | train_error: 1.69 | train_acc: 0.837 
+
    +
    +
      [================>-----------------------] 42.60% | train_error: 1.69 | train_acc: 0.837 
+
    +
    +
      [================>-----------------------] 42.70% | train_error: 1.68 | train_acc: 0.838 
+
    +
    +
      [================>-----------------------] 42.80% | train_error: 1.68 | train_acc: 0.838 
+
    +
    +
      [================>-----------------------] 42.90% | train_error: 1.68 | train_acc: 0.838 
+
    +
    +
      [================>-----------------------] 43.00% | train_error: 1.68 | train_acc: 0.838 
+
    +
    +
      [================>-----------------------] 43.10% | train_error: 1.68 | train_acc: 0.838 
+
    +
    +
      [================>-----------------------] 43.20% | train_error: 1.67 | train_acc: 0.839 
+
    +
    +
      [================>-----------------------] 43.30% | train_error: 1.67 | train_acc: 0.839 
+
    +
    +
      [================>-----------------------] 43.40% | train_error: 1.67 | train_acc: 0.839 
+
    +
    +
      [================>-----------------------] 43.50% | train_error: 1.67 | train_acc: 0.839 
+
    +
    +
      [================>-----------------------] 43.60% | train_error: 1.66 | train_acc: 0.840 
+
    +
    +
      [================>-----------------------] 43.70% | train_error: 1.66 | train_acc: 0.840 
+
    +
    +
      [================>-----------------------] 43.80% | train_error: 1.65 | train_acc: 0.840 
+
    +
    +
      [================>-----------------------] 43.90% | train_error: 1.65 | train_acc: 0.841 
+
    +
    +
      [================>-----------------------] 44.00% | train_error: 1.65 | train_acc: 0.841 
+
    +
    +
      [================>-----------------------] 44.10% | train_error: 1.64 | train_acc: 0.841 
+
    +
    +
      [================>-----------------------] 44.20% | train_error: 1.64 | train_acc: 0.842 
+
    +
    +
      [================>-----------------------] 44.30% | train_error: 1.62 | train_acc: 0.843 
+
    +
    +
      [================>-----------------------] 44.40% | train_error: 1.62 | train_acc: 0.843 
+
    +
    +
      [================>-----------------------] 44.50% | train_error: 1.62 | train_acc: 0.843 
+
    +
    +
      [================>-----------------------] 44.60% | train_error: 1.62 | train_acc: 0.844 
+
    +
    +
      [================>-----------------------] 44.70% | train_error: 1.61 | train_acc: 0.844 
+
    +
    +
      [================>-----------------------] 44.80% | train_error: 1.61 | train_acc: 0.845 
+
    +
    +
      [================>-----------------------] 44.90% | train_error: 1.60 | train_acc: 0.846 
+
    +
    +
      [=================>----------------------] 45.00% | train_error: 1.59 | train_acc: 0.846 
+
    +
    +
      [=================>----------------------] 45.10% | train_error: 1.59 | train_acc: 0.846 
+
    +
    +
      [=================>----------------------] 45.20% | train_error: 1.59 | train_acc: 0.847 
+
    +
    +
      [=================>----------------------] 45.30% | train_error: 1.58 | train_acc: 0.847 
+
    +
    +
      [=================>----------------------] 45.40% | train_error: 1.58 | train_acc: 0.847 
+
    +
    +
      [=================>----------------------] 45.50% | train_error: 1.58 | train_acc: 0.848 
+
    +
    +
      [=================>----------------------] 45.60% | train_error: 1.57 | train_acc: 0.848 
+
    +
    +
      [=================>----------------------] 45.70% | train_error: 1.57 | train_acc: 0.849 
+
    +
    +
      [=================>----------------------] 45.80% | train_error: 1.56 | train_acc: 0.849 
+
    +
    +
      [=================>----------------------] 45.90% | train_error: 1.56 | train_acc: 0.850 
+
    +
    +
      [=================>----------------------] 46.00% | train_error: 1.55 | train_acc: 0.850 
+
    +
    +
      [=================>----------------------] 46.10% | train_error: 1.55 | train_acc: 0.851 
+
    +
    +
      [=================>----------------------] 46.20% | train_error: 1.55 | train_acc: 0.851 
+
    +
    +
      [=================>----------------------] 46.30% | train_error: 1.54 | train_acc: 0.852 
+
    +
    +
      [=================>----------------------] 46.40% | train_error: 1.53 | train_acc: 0.852 
+
    +
    +
      [=================>----------------------] 46.50% | train_error: 1.53 | train_acc: 0.852 
+
    +
    +
      [=================>----------------------] 46.60% | train_error: 1.53 | train_acc: 0.853 
+
    +
    +
      [=================>----------------------] 46.70% | train_error: 1.52 | train_acc: 0.853 
+
    +
    +
      [=================>----------------------] 46.80% | train_error: 1.52 | train_acc: 0.854 
+
    +
    +
      [=================>----------------------] 46.90% | train_error: 1.52 | train_acc: 0.854 
+
    +
    +
      [=================>----------------------] 47.00% | train_error: 1.51 | train_acc: 0.854 
+
    +
    +
      [=================>----------------------] 47.10% | train_error: 1.51 | train_acc: 0.854 
+
    +
    +
      [=================>----------------------] 47.20% | train_error: 1.50 | train_acc: 0.855 
+
    +
    +
      [=================>----------------------] 47.30% | train_error: 1.50 | train_acc: 0.855 
+
    +
    +
      [=================>----------------------] 47.40% | train_error: 1.49 | train_acc: 0.856 
+
    +
    +
      [==================>---------------------] 47.50% | train_error: 1.47 | train_acc: 0.858 
+
    +
    +
      [==================>---------------------] 47.60% | train_error: 1.47 | train_acc: 0.858 
+
    +
    +
      [==================>---------------------] 47.70% | train_error: 1.47 | train_acc: 0.858 
+
    +
    +
      [==================>---------------------] 47.80% | train_error: 1.46 | train_acc: 0.859 
+
    +
    +
      [==================>---------------------] 47.90% | train_error: 1.46 | train_acc: 0.859 
+
    +
    +
      [==================>---------------------] 48.00% | train_error: 1.46 | train_acc: 0.859 
+
    +
    +
      [==================>---------------------] 48.10% | train_error: 1.46 | train_acc: 0.859 
+
    +
    +
      [==================>---------------------] 48.20% | train_error: 1.46 | train_acc: 0.859 
+
    +
    +
      [==================>---------------------] 48.30% | train_error: 1.46 | train_acc: 0.859 
+
    +
    +
      [==================>---------------------] 48.40% | train_error: 1.45 | train_acc: 0.860 
+
    +
    +
      [==================>---------------------] 48.50% | train_error: 1.44 | train_acc: 0.861 
+
    +
    +
      [==================>---------------------] 48.60% | train_error: 1.44 | train_acc: 0.861 
+
    +
    +
      [==================>---------------------] 48.70% | train_error: 1.44 | train_acc: 0.861 
+
    +
    +
      [==================>---------------------] 48.80% | train_error: 1.44 | train_acc: 0.861 
+
    +
    +
      [==================>---------------------] 48.90% | train_error: 1.43 | train_acc: 0.862 
+
    +
    +
      [==================>---------------------] 49.00% | train_error: 1.42 | train_acc: 0.863 
+
    +
    +
      [==================>---------------------] 49.10% | train_error: 1.41 | train_acc: 0.864 
+
    +
    +
      [==================>---------------------] 49.20% | train_error: 1.40 | train_acc: 0.865 
+
    +
    +
      [==================>---------------------] 49.30% | train_error: 1.39 | train_acc: 0.866 
+
    +
    +
      [==================>---------------------] 49.40% | train_error: 1.39 | train_acc: 0.866 
+
    +
    +
      [==================>---------------------] 49.50% | train_error: 1.39 | train_acc: 0.866 
+
    +
    +
      [==================>---------------------] 49.60% | train_error: 1.39 | train_acc: 0.866 
+
    +
    +
      [==================>---------------------] 49.70% | train_error: 1.38 | train_acc: 0.867 
+
    +
    +
      [==================>---------------------] 49.80% | train_error: 1.38 | train_acc: 0.867 
+
    +
    +
      [==================>---------------------] 49.90% | train_error: 1.37 | train_acc: 0.868 
+
    +
    +
      [===================>--------------------] 50.00% | train_error: 1.36 | train_acc: 0.869 
+
    +
    +
      [===================>--------------------] 50.10% | train_error: 1.35 | train_acc: 0.869 
+
    +
    +
      [===================>--------------------] 50.20% | train_error: 1.35 | train_acc: 0.870 
+
    +
    +
      [===================>--------------------] 50.30% | train_error: 1.34 | train_acc: 0.870 
+
    +
    +
      [===================>--------------------] 50.40% | train_error: 1.34 | train_acc: 0.870 
+
    +
    +
      [===================>--------------------] 50.50% | train_error: 1.34 | train_acc: 0.870 
+
    +
    +
      [===================>--------------------] 50.60% | train_error: 1.34 | train_acc: 0.871 
+
    +
    +
      [===================>--------------------] 50.70% | train_error: 1.33 | train_acc: 0.872 
+
    +
    +
      [===================>--------------------] 50.80% | train_error: 1.32 | train_acc: 0.873 
+
    +
    +
      [===================>--------------------] 50.90% | train_error: 1.31 | train_acc: 0.873 
+
    +
    +
      [===================>--------------------] 51.00% | train_error: 1.31 | train_acc: 0.874 
+
    +
    +
      [===================>--------------------] 51.10% | train_error: 1.31 | train_acc: 0.874 
+
    +
    +
      [===================>--------------------] 51.20% | train_error: 1.31 | train_acc: 0.874 
+
    +
    +
      [===================>--------------------] 51.30% | train_error: 1.31 | train_acc: 0.874 
+
    +
    +
      [===================>--------------------] 51.40% | train_error: 1.31 | train_acc: 0.874 
+
    +
    +
      [===================>--------------------] 51.50% | train_error: 1.31 | train_acc: 0.874 
+
    +
    +
      [===================>--------------------] 51.60% | train_error: 1.30 | train_acc: 0.874 
+
    +
    +
      [===================>--------------------] 51.70% | train_error: 1.30 | train_acc: 0.875 
+
    +
    +
      [===================>--------------------] 51.80% | train_error: 1.30 | train_acc: 0.875 
+
    +
    +
      [===================>--------------------] 51.90% | train_error: 1.29 | train_acc: 0.875 
+
    +
    +
      [===================>--------------------] 52.00% | train_error: 1.29 | train_acc: 0.875 
+
    +
    +
      [===================>--------------------] 52.10% | train_error: 1.29 | train_acc: 0.875 
+
    +
    +
      [===================>--------------------] 52.20% | train_error: 1.29 | train_acc: 0.875 
+
    +
    +
      [===================>--------------------] 52.30% | train_error: 1.29 | train_acc: 0.876 
+
    +
    +
      [===================>--------------------] 52.40% | train_error: 1.28 | train_acc: 0.876 
+
    +
    +
      [====================>-------------------] 52.50% | train_error: 1.28 | train_acc: 0.876 
+
    +
    +
      [====================>-------------------] 52.60% | train_error: 1.28 | train_acc: 0.877 
+
    +
    +
      [====================>-------------------] 52.70% | train_error: 1.28 | train_acc: 0.877 
+
    +
    +
      [====================>-------------------] 52.80% | train_error: 1.27 | train_acc: 0.877 
+
    +
    +
      [====================>-------------------] 52.90% | train_error: 1.27 | train_acc: 0.877 
+
    +
    +
      [====================>-------------------] 53.00% | train_error: 1.27 | train_acc: 0.877 
+
    +
    +
      [====================>-------------------] 53.10% | train_error: 1.27 | train_acc: 0.878 
+
    +
    +
      [====================>-------------------] 53.20% | train_error: 1.26 | train_acc: 0.878 
+
    +
    +
      [====================>-------------------] 53.30% | train_error: 1.26 | train_acc: 0.878 
+
    +
    +
      [====================>-------------------] 53.40% | train_error: 1.26 | train_acc: 0.879 
+
    +
    +
      [====================>-------------------] 53.50% | train_error: 1.25 | train_acc: 0.879 
+
    +
    +
      [====================>-------------------] 53.60% | train_error: 1.25 | train_acc: 0.880 
+
    +
    +
      [====================>-------------------] 53.70% | train_error: 1.24 | train_acc: 0.880 
+
    +
    +
      [====================>-------------------] 53.80% | train_error: 1.24 | train_acc: 0.880 
+
    +
    +
      [====================>-------------------] 53.90% | train_error: 1.24 | train_acc: 0.880 
+
    +
    +
      [====================>-------------------] 54.00% | train_error: 1.23 | train_acc: 0.881 
+
    +
    +
      [====================>-------------------] 54.10% | train_error: 1.22 | train_acc: 0.882 
+
    +
    +
      [====================>-------------------] 54.20% | train_error: 1.22 | train_acc: 0.882 
+
    +
    +
      [====================>-------------------] 54.30% | train_error: 1.21 | train_acc: 0.883 
+
    +
    +
      [====================>-------------------] 54.40% | train_error: 1.21 | train_acc: 0.883 
+
    +
    +
      [====================>-------------------] 54.50% | train_error: 1.21 | train_acc: 0.883 
+
    +
    +
      [====================>-------------------] 54.60% | train_error: 1.21 | train_acc: 0.883 
+
    +
    +
      [====================>-------------------] 54.70% | train_error: 1.21 | train_acc: 0.884 
+
    +
    +
      [====================>-------------------] 54.80% | train_error: 1.20 | train_acc: 0.884 
+
    +
    +
      [====================>-------------------] 54.90% | train_error: 1.20 | train_acc: 0.884 
+
    +
    +
      [=====================>------------------] 55.00% | train_error: 1.20 | train_acc: 0.884 
+
    +
    +
      [=====================>------------------] 55.10% | train_error: 1.20 | train_acc: 0.884 
+
    +
    +
      [=====================>------------------] 55.20% | train_error: 1.19 | train_acc: 0.885 
+
    +
    +
      [=====================>------------------] 55.30% | train_error: 1.19 | train_acc: 0.885 
+
    +
    +
      [=====================>------------------] 55.40% | train_error: 1.19 | train_acc: 0.885 
+
    +
    +
      [=====================>------------------] 55.50% | train_error: 1.19 | train_acc: 0.885 
+
    +
    +
      [=====================>------------------] 55.60% | train_error: 1.19 | train_acc: 0.885 
+
    +
    +
      [=====================>------------------] 55.70% | train_error: 1.19 | train_acc: 0.885 
+
    +
    +
      [=====================>------------------] 55.80% | train_error: 1.18 | train_acc: 0.886 
+
    +
    +
      [=====================>------------------] 55.90% | train_error: 1.19 | train_acc: 0.885 
+
    +
    +
      [=====================>------------------] 56.00% | train_error: 1.19 | train_acc: 0.885 
+
    +
    +
      [=====================>------------------] 56.10% | train_error: 1.18 | train_acc: 0.886 
+
    +
    +
      [=====================>------------------] 56.20% | train_error: 1.18 | train_acc: 0.886 
+
    +
    +
      [=====================>------------------] 56.30% | train_error: 1.17 | train_acc: 0.887 
+
    +
    +
      [=====================>------------------] 56.40% | train_error: 1.17 | train_acc: 0.887 
+
    +
    +
      [=====================>------------------] 56.50% | train_error: 1.17 | train_acc: 0.887 
+
    +
    +
      [=====================>------------------] 56.60% | train_error: 1.16 | train_acc: 0.888 
+
    +
    +
      [=====================>------------------] 56.70% | train_error: 1.16 | train_acc: 0.888 
+
    +
    +
      [=====================>------------------] 56.80% | train_error: 1.15 | train_acc: 0.889 
+
    +
    +
      [=====================>------------------] 56.90% | train_error: 1.15 | train_acc: 0.889 
+
    +
    +
      [=====================>------------------] 57.00% | train_error: 1.14 | train_acc: 0.890 
+
    +
    +
      [=====================>------------------] 57.10% | train_error: 1.14 | train_acc: 0.890 
+
    +
    +
      [=====================>------------------] 57.20% | train_error: 1.14 | train_acc: 0.890 
+
    +
    +
      [=====================>------------------] 57.30% | train_error: 1.14 | train_acc: 0.890 
+
    +
    +
      [=====================>------------------] 57.40% | train_error: 1.13 | train_acc: 0.891 
+
    +
    +
      [======================>-----------------] 57.50% | train_error: 1.13 | train_acc: 0.891 
+
    +
    +
      [======================>-----------------] 57.60% | train_error: 1.13 | train_acc: 0.891 
+
    +
    +
      [======================>-----------------] 57.70% | train_error: 1.12 | train_acc: 0.891 
+
    +
    +
      [======================>-----------------] 57.80% | train_error: 1.13 | train_acc: 0.891 
+
    +
    +
      [======================>-----------------] 57.90% | train_error: 1.12 | train_acc: 0.892 
+
    +
    +
      [======================>-----------------] 58.00% | train_error: 1.11 | train_acc: 0.893 
+
    +
    +
      [======================>-----------------] 58.10% | train_error: 1.11 | train_acc: 0.893 
+
    +
    +
      [======================>-----------------] 58.20% | train_error: 1.09 | train_acc: 0.894 
+
    +
    +
      [======================>-----------------] 58.30% | train_error: 1.09 | train_acc: 0.895 
+
    +
    +
      [======================>-----------------] 58.40% | train_error: 1.09 | train_acc: 0.895 
+
    +
    +
      [======================>-----------------] 58.50% | train_error: 1.09 | train_acc: 0.895 
+
    +
    +
      [======================>-----------------] 58.60% | train_error: 1.09 | train_acc: 0.895 
+
    +
    +
      [======================>-----------------] 58.70% | train_error: 1.09 | train_acc: 0.895 
+
    +
    +
      [======================>-----------------] 58.80% | train_error: 1.08 | train_acc: 0.895 
+
    +
    +
      [======================>-----------------] 58.90% | train_error: 1.08 | train_acc: 0.895 
+
    +
    +
      [======================>-----------------] 59.00% | train_error: 1.08 | train_acc: 0.896 
+
    +
    +
      [======================>-----------------] 59.10% | train_error: 1.08 | train_acc: 0.896 
+
    +
    +
      [======================>-----------------] 59.20% | train_error: 1.08 | train_acc: 0.896 
+
    +
    +
      [======================>-----------------] 59.30% | train_error: 1.07 | train_acc: 0.896 
+
    +
    +
      [======================>-----------------] 59.40% | train_error: 1.07 | train_acc: 0.897 
+
    +
    +
      [======================>-----------------] 59.50% | train_error: 1.06 | train_acc: 0.898 
+
    +
    +
      [======================>-----------------] 59.60% | train_error: 1.05 | train_acc: 0.898 
+
    +
    +
      [======================>-----------------] 59.70% | train_error: 1.05 | train_acc: 0.899 
+
    +
    +
      [======================>-----------------] 59.80% | train_error: 1.04 | train_acc: 0.899 
+
    +
    +
      [======================>-----------------] 59.90% | train_error: 1.04 | train_acc: 0.899 
+
    +
    +
      [=======================>----------------] 60.00% | train_error: 1.04 | train_acc: 0.899 
+
    +
    +
      [=======================>----------------] 60.10% | train_error: 1.04 | train_acc: 0.900 
+
    +
    +
      [=======================>----------------] 60.20% | train_error: 1.03 | train_acc: 0.900 
+
    +
    +
      [=======================>----------------] 60.30% | train_error: 1.03 | train_acc: 0.901 
+
    +
    +
      [=======================>----------------] 60.40% | train_error: 1.03 | train_acc: 0.901 
+
    +
    +
      [=======================>----------------] 60.50% | train_error: 1.02 | train_acc: 0.901 
+
    +
    +
      [=======================>----------------] 60.60% | train_error: 1.02 | train_acc: 0.902 
+
    +
    +
      [=======================>----------------] 60.70% | train_error: 1.01 | train_acc: 0.902 
+
    +
    +
      [=======================>----------------] 60.80% | train_error: 1.01 | train_acc: 0.903 
+
    +
    +
      [=======================>----------------] 60.90% | train_error: 1.00 | train_acc: 0.903 
+
    +
    +
      [=======================>----------------] 61.00% | train_error: 1.00 | train_acc: 0.903 
+
    +
    +
      [=======================>----------------] 61.10% | train_error: 1.00 | train_acc: 0.903 
+
    +
    +
      [=======================>----------------] 61.20% | train_error: 1.00 | train_acc: 0.903 
+
    +
    +
      [=======================>----------------] 61.30% | train_error: 0.999 | train_acc: 0.904 
+
    +
    +
      [=======================>----------------] 61.40% | train_error: 0.998 | train_acc: 0.904 
+
    +
    +
      [=======================>----------------] 61.50% | train_error: 0.998 | train_acc: 0.904 
+
    +
    +
      [=======================>----------------] 61.60% | train_error: 0.994 | train_acc: 0.904 
+
    +
    +
      [=======================>----------------] 61.70% | train_error: 0.994 | train_acc: 0.904 
+
    +
    +
      [=======================>----------------] 61.80% | train_error: 0.991 | train_acc: 0.904 
+
    +
    +
      [=======================>----------------] 61.90% | train_error: 0.987 | train_acc: 0.905 
+
    +
    +
      [=======================>----------------] 62.00% | train_error: 0.986 | train_acc: 0.905 
+
    +
    +
      [=======================>----------------] 62.10% | train_error: 0.978 | train_acc: 0.906 
+
    +
    +
      [=======================>----------------] 62.20% | train_error: 0.976 | train_acc: 0.906 
+
    +
    +
      [=======================>----------------] 62.30% | train_error: 0.976 | train_acc: 0.906 
+
    +
    +
      [=======================>----------------] 62.40% | train_error: 0.973 | train_acc: 0.906 
+
    +
    +
      [========================>---------------] 62.50% | train_error: 0.972 | train_acc: 0.906 
+
    +
    +
      [========================>---------------] 62.60% | train_error: 0.968 | train_acc: 0.907 
+
    +
    +
      [========================>---------------] 62.70% | train_error: 0.968 | train_acc: 0.907 
+
    +
    +
      [========================>---------------] 62.80% | train_error: 0.965 | train_acc: 0.907 
+
    +
    +
      [========================>---------------] 62.90% | train_error: 0.966 | train_acc: 0.907 
+
    +
    +
      [========================>---------------] 63.00% | train_error: 0.962 | train_acc: 0.907 
+
    +
    +
      [========================>---------------] 63.10% | train_error: 0.954 | train_acc: 0.908 
+
    +
    +
      [========================>---------------] 63.20% | train_error: 0.951 | train_acc: 0.908 
+
    +
    +
      [========================>---------------] 63.30% | train_error: 0.948 | train_acc: 0.909 
+
    +
    +
      [========================>---------------] 63.40% | train_error: 0.946 | train_acc: 0.909 
+
    +
    +
      [========================>---------------] 63.50% | train_error: 0.943 | train_acc: 0.909 
+
    +
    +
      [========================>---------------] 63.60% | train_error: 0.935 | train_acc: 0.910 
+
    +
    +
      [========================>---------------] 63.70% | train_error: 0.926 | train_acc: 0.911 
+
    +
    +
      [========================>---------------] 63.80% | train_error: 0.926 | train_acc: 0.911 
+
    +
    +
      [========================>---------------] 63.90% | train_error: 0.926 | train_acc: 0.911 
+
    +
    +
      [========================>---------------] 64.00% | train_error: 0.914 | train_acc: 0.912 
+
    +
    +
      [========================>---------------] 64.10% | train_error: 0.914 | train_acc: 0.912 
+
    +
    +
      [========================>---------------] 64.20% | train_error: 0.906 | train_acc: 0.913 
+
    +
    +
      [========================>---------------] 64.30% | train_error: 0.903 | train_acc: 0.913 
+
    +
    +
      [========================>---------------] 64.40% | train_error: 0.900 | train_acc: 0.913 
+
    +
    +
      [========================>---------------] 64.50% | train_error: 0.895 | train_acc: 0.914 
+
    +
    +
      [========================>---------------] 64.60% | train_error: 0.894 | train_acc: 0.914 
+
    +
    +
      [========================>---------------] 64.70% | train_error: 0.891 | train_acc: 0.914 
+
    +
    +
      [========================>---------------] 64.80% | train_error: 0.893 | train_acc: 0.914 
+
    +
    +
      [========================>---------------] 64.90% | train_error: 0.893 | train_acc: 0.914 
+
    +
    +
      [=========================>--------------] 65.00% | train_error: 0.890 | train_acc: 0.914 
+
    +
    +
      [=========================>--------------] 65.10% | train_error: 0.889 | train_acc: 0.914 
+
    +
    +
      [=========================>--------------] 65.20% | train_error: 0.883 | train_acc: 0.915 
+
    +
    +
      [=========================>--------------] 65.30% | train_error: 0.880 | train_acc: 0.915 
+
    +
    +
      [=========================>--------------] 65.40% | train_error: 0.878 | train_acc: 0.915 
+
    +
    +
      [=========================>--------------] 65.50% | train_error: 0.876 | train_acc: 0.915 
+
    +
    +
      [=========================>--------------] 65.60% | train_error: 0.875 | train_acc: 0.916 
+
    +
    +
      [=========================>--------------] 65.70% | train_error: 0.874 | train_acc: 0.916 
+
    +
    +
      [=========================>--------------] 65.80% | train_error: 0.875 | train_acc: 0.916 
+
    +
    +
      [=========================>--------------] 65.90% | train_error: 0.873 | train_acc: 0.916 
+
    +
    +
      [=========================>--------------] 66.00% | train_error: 0.871 | train_acc: 0.916 
+
    +
    +
      [=========================>--------------] 66.10% | train_error: 0.868 | train_acc: 0.916 
+
    +
    +
      [=========================>--------------] 66.20% | train_error: 0.866 | train_acc: 0.916 
+
    +
    +
      [=========================>--------------] 66.30% | train_error: 0.864 | train_acc: 0.917 
+
    +
    +
      [=========================>--------------] 66.40% | train_error: 0.855 | train_acc: 0.918 
+
    +
    +
      [=========================>--------------] 66.50% | train_error: 0.851 | train_acc: 0.918 
+
    +
    +
      [=========================>--------------] 66.60% | train_error: 0.850 | train_acc: 0.918 
+
    +
    +
      [=========================>--------------] 66.70% | train_error: 0.850 | train_acc: 0.918 
+
    +
    +
      [=========================>--------------] 66.80% | train_error: 0.848 | train_acc: 0.918 
+
    +
    +
      [=========================>--------------] 66.90% | train_error: 0.844 | train_acc: 0.919 
+
    +
    +
      [=========================>--------------] 67.00% | train_error: 0.845 | train_acc: 0.918 
+
    +
    +
      [=========================>--------------] 67.10% | train_error: 0.842 | train_acc: 0.919 
+
    +
    +
      [=========================>--------------] 67.20% | train_error: 0.841 | train_acc: 0.919 
+
    +
    +
      [=========================>--------------] 67.30% | train_error: 0.841 | train_acc: 0.919 
+
    +
    +
      [=========================>--------------] 67.40% | train_error: 0.836 | train_acc: 0.919 
+
    +
    +
      [==========================>-------------] 67.50% | train_error: 0.821 | train_acc: 0.921 
+
    +
    +
      [==========================>-------------] 67.60% | train_error: 0.808 | train_acc: 0.922 
+
    +
    +
      [==========================>-------------] 67.70% | train_error: 0.805 | train_acc: 0.922 
+
    +
    +
      [==========================>-------------] 67.80% | train_error: 0.800 | train_acc: 0.923 
+
    +
    +
      [==========================>-------------] 67.90% | train_error: 0.799 | train_acc: 0.923 
+
    +
    +
      [==========================>-------------] 68.00% | train_error: 0.792 | train_acc: 0.924 
+
    +
    +
      [==========================>-------------] 68.10% | train_error: 0.791 | train_acc: 0.924 
+
    +
    +
      [==========================>-------------] 68.20% | train_error: 0.782 | train_acc: 0.925 
+
    +
    +
      [==========================>-------------] 68.30% | train_error: 0.774 | train_acc: 0.925 
+
    +
    +
      [==========================>-------------] 68.40% | train_error: 0.766 | train_acc: 0.926 
+
    +
    +
      [==========================>-------------] 68.50% | train_error: 0.763 | train_acc: 0.926 
+
    +
    +
      [==========================>-------------] 68.60% | train_error: 0.757 | train_acc: 0.927 
+
    +
    +
      [==========================>-------------] 68.70% | train_error: 0.753 | train_acc: 0.927 
+
    +
    +
      [==========================>-------------] 68.80% | train_error: 0.754 | train_acc: 0.927 
+
    +
    +
      [==========================>-------------] 68.90% | train_error: 0.747 | train_acc: 0.928 
+
    +
    +
      [==========================>-------------] 69.00% | train_error: 0.740 | train_acc: 0.929 
+
    +
    +
      [==========================>-------------] 69.10% | train_error: 0.746 | train_acc: 0.928 
+
    +
    +
      [==========================>-------------] 69.20% | train_error: 0.737 | train_acc: 0.929 
+
    +
    +
      [==========================>-------------] 69.30% | train_error: 0.744 | train_acc: 0.928 
+
    +
    +
      [==========================>-------------] 69.40% | train_error: 0.736 | train_acc: 0.929 
+
    +
    +
      [==========================>-------------] 69.50% | train_error: 0.745 | train_acc: 0.928 
+
    +
    +
      [==========================>-------------] 69.60% | train_error: 0.737 | train_acc: 0.929 
+
    +
    +
      [==========================>-------------] 69.70% | train_error: 0.736 | train_acc: 0.929 
+
    +
    +
      [==========================>-------------] 69.80% | train_error: 0.724 | train_acc: 0.930 
+
    +
    +
      [==========================>-------------] 69.90% | train_error: 0.722 | train_acc: 0.930 
+
    +
    +
      [===========================>------------] 70.00% | train_error: 0.718 | train_acc: 0.931 
+
    +
    +
      [===========================>------------] 70.10% | train_error: 0.718 | train_acc: 0.931 
+
    +
    +
      [===========================>------------] 70.20% | train_error: 0.717 | train_acc: 0.931 
+
    +
    +
      [===========================>------------] 70.30% | train_error: 0.712 | train_acc: 0.931 
+
    +
    +
      [===========================>------------] 70.40% | train_error: 0.713 | train_acc: 0.931 
+
    +
    +
      [===========================>------------] 70.50% | train_error: 0.710 | train_acc: 0.931 
+
    +
    +
      [===========================>------------] 70.60% | train_error: 0.708 | train_acc: 0.932 
+
    +
    +
      [===========================>------------] 70.70% | train_error: 0.705 | train_acc: 0.932 
+
    +
    +
      [===========================>------------] 70.80% | train_error: 0.702 | train_acc: 0.932 
+
    +
    +
      [===========================>------------] 70.90% | train_error: 0.701 | train_acc: 0.932 
+
    +
    +
      [===========================>------------] 71.00% | train_error: 0.695 | train_acc: 0.933 
+
    +
    +
      [===========================>------------] 71.10% | train_error: 0.694 | train_acc: 0.933 
+
    +
    +
      [===========================>------------] 71.20% | train_error: 0.691 | train_acc: 0.933 
+
    +
    +
      [===========================>------------] 71.30% | train_error: 0.687 | train_acc: 0.934 
+
    +
    +
      [===========================>------------] 71.40% | train_error: 0.683 | train_acc: 0.934 
+
    +
    +
      [===========================>------------] 71.50% | train_error: 0.682 | train_acc: 0.934 
+
    +
    +
      [===========================>------------] 71.60% | train_error: 0.677 | train_acc: 0.935 
+
    +
    +
      [===========================>------------] 71.70% | train_error: 0.672 | train_acc: 0.935 
+
    +
    +
      [===========================>------------] 71.80% | train_error: 0.669 | train_acc: 0.935 
+
    +
    +
      [===========================>------------] 71.90% | train_error: 0.669 | train_acc: 0.935 
+
    +
    +
      [===========================>------------] 72.00% | train_error: 0.668 | train_acc: 0.936 
+
    +
    +
      [===========================>------------] 72.10% | train_error: 0.665 | train_acc: 0.936 
+
    +
    +
      [===========================>------------] 72.20% | train_error: 0.658 | train_acc: 0.936 
+
    +
    +
      [===========================>------------] 72.30% | train_error: 0.655 | train_acc: 0.937 
+
    +
    +
      [===========================>------------] 72.40% | train_error: 0.654 | train_acc: 0.937 
+
    +
    +
      [============================>-----------] 72.50% | train_error: 0.657 | train_acc: 0.937 
+
    +
    +
      [============================>-----------] 72.60% | train_error: 0.652 | train_acc: 0.937 
+
    +
    +
      [============================>-----------] 72.70% | train_error: 0.647 | train_acc: 0.938 
+
    +
    +
      [============================>-----------] 72.80% | train_error: 0.645 | train_acc: 0.938 
+
    +
    +
      [============================>-----------] 72.90% | train_error: 0.645 | train_acc: 0.938 
+
    +
    +
      [============================>-----------] 73.00% | train_error: 0.630 | train_acc: 0.939 
+
    +
    +
      [============================>-----------] 73.10% | train_error: 0.637 | train_acc: 0.939 
+
    +
    +
      [============================>-----------] 73.20% | train_error: 0.624 | train_acc: 0.940 
+
    +
    +
      [============================>-----------] 73.30% | train_error: 0.630 | train_acc: 0.939 
+
    +
    +
      [============================>-----------] 73.40% | train_error: 0.618 | train_acc: 0.940 
+
    +
    +
      [============================>-----------] 73.50% | train_error: 0.623 | train_acc: 0.940 
+
    +
    +
      [============================>-----------] 73.60% | train_error: 0.611 | train_acc: 0.941 
+
    +
    +
      [============================>-----------] 73.70% | train_error: 0.612 | train_acc: 0.941 
+
    +
    +
      [============================>-----------] 73.80% | train_error: 0.614 | train_acc: 0.941 
+
    +
    +
      [============================>-----------] 73.90% | train_error: 0.608 | train_acc: 0.941 
+
    +
    +
      [============================>-----------] 74.00% | train_error: 0.605 | train_acc: 0.942 
+
    +
    +
      [============================>-----------] 74.10% | train_error: 0.609 | train_acc: 0.941 
+
    +
    +
      [============================>-----------] 74.20% | train_error: 0.604 | train_acc: 0.942 
+
    +
    +
      [============================>-----------] 74.30% | train_error: 0.601 | train_acc: 0.942 
+
    +
    +
      [============================>-----------] 74.40% | train_error: 0.600 | train_acc: 0.942 
+
    +
    +
      [============================>-----------] 74.50% | train_error: 0.601 | train_acc: 0.942 
+
    +
    +
      [============================>-----------] 74.60% | train_error: 0.599 | train_acc: 0.942 
+
    +
    +
      [============================>-----------] 74.70% | train_error: 0.593 | train_acc: 0.943 
+
    +
    +
      [============================>-----------] 74.80% | train_error: 0.592 | train_acc: 0.943 
+
    +
    +
      [============================>-----------] 74.90% | train_error: 0.593 | train_acc: 0.943 
+
    +
    +
      [=============================>----------] 75.00% | train_error: 0.589 | train_acc: 0.943 
+
    +
    +
      [=============================>----------] 75.10% | train_error: 0.590 | train_acc: 0.943 
+
    +
    +
      [=============================>----------] 75.20% | train_error: 0.585 | train_acc: 0.944 
+
    +
    +
      [=============================>----------] 75.30% | train_error: 0.590 | train_acc: 0.943 
+
    +
    +
      [=============================>----------] 75.40% | train_error: 0.580 | train_acc: 0.944 
+
    +
    +
      [=============================>----------] 75.50% | train_error: 0.579 | train_acc: 0.944 
+
    +
    +
      [=============================>----------] 75.60% | train_error: 0.574 | train_acc: 0.945 
+
    +
    +
      [=============================>----------] 75.70% | train_error: 0.578 | train_acc: 0.944 
+
    +
    +
      [=============================>----------] 75.80% | train_error: 0.560 | train_acc: 0.946 
+
    +
    +
      [=============================>----------] 75.90% | train_error: 0.566 | train_acc: 0.945 
+
    +
    +
      [=============================>----------] 76.00% | train_error: 0.564 | train_acc: 0.946 
+
    +
    +
      [=============================>----------] 76.10% | train_error: 0.563 | train_acc: 0.946 
+
    +
    +
      [=============================>----------] 76.20% | train_error: 0.559 | train_acc: 0.946 
+
    +
    +
      [=============================>----------] 76.30% | train_error: 0.560 | train_acc: 0.946 
+
    +
    +
      [=============================>----------] 76.40% | train_error: 0.549 | train_acc: 0.947 
+
    +
    +
      [=============================>----------] 76.50% | train_error: 0.563 | train_acc: 0.946 
+
    +
    +
      [=============================>----------] 76.60% | train_error: 0.542 | train_acc: 0.948 
+
    +
    +
      [=============================>----------] 76.70% | train_error: 0.558 | train_acc: 0.946 
+
    +
    +
      [=============================>----------] 76.80% | train_error: 0.542 | train_acc: 0.948 
+
    +
    +
      [=============================>----------] 76.90% | train_error: 0.556 | train_acc: 0.946 
+
    +
    +
      [=============================>----------] 77.00% | train_error: 0.536 | train_acc: 0.948 
+
    +
    +
      [=============================>----------] 77.10% | train_error: 0.549 | train_acc: 0.947 
+
    +
    +
      [=============================>----------] 77.20% | train_error: 0.529 | train_acc: 0.949 
+
    +
    +
      [=============================>----------] 77.30% | train_error: 0.536 | train_acc: 0.948 
+
    +
    +
      [=============================>----------] 77.40% | train_error: 0.534 | train_acc: 0.949 
+
    +
    +
      [==============================>---------] 77.50% | train_error: 0.532 | train_acc: 0.949 
+
    +
    +
      [==============================>---------] 77.60% | train_error: 0.528 | train_acc: 0.949 
+
    +
    +
      [==============================>---------] 77.70% | train_error: 0.526 | train_acc: 0.949 
+
    +
    +
      [==============================>---------] 77.80% | train_error: 0.515 | train_acc: 0.950 
+
    +
    +
      [==============================>---------] 77.90% | train_error: 0.517 | train_acc: 0.950 
+
    +
    +
      [==============================>---------] 78.00% | train_error: 0.510 | train_acc: 0.951 
+
    +
    +
      [==============================>---------] 78.10% | train_error: 0.511 | train_acc: 0.951 
+
    +
    +
      [==============================>---------] 78.20% | train_error: 0.510 | train_acc: 0.951 
+
    +
    +
      [==============================>---------] 78.30% | train_error: 0.504 | train_acc: 0.951 
+
    +
    +
      [==============================>---------] 78.40% | train_error: 0.510 | train_acc: 0.951 
+
    +
    +
      [==============================>---------] 78.50% | train_error: 0.507 | train_acc: 0.951 
+
    +
    +
      [==============================>---------] 78.60% | train_error: 0.509 | train_acc: 0.951 
+
    +
    +
      [==============================>---------] 78.70% | train_error: 0.499 | train_acc: 0.952 
+
    +
    +
      [==============================>---------] 78.80% | train_error: 0.506 | train_acc: 0.951 
+
    +
    +
      [==============================>---------] 78.90% | train_error: 0.499 | train_acc: 0.952 
+
    +
    +
      [==============================>---------] 79.00% | train_error: 0.504 | train_acc: 0.951 
+
    +
    +
      [==============================>---------] 79.10% | train_error: 0.503 | train_acc: 0.952 
+
    +
    +
      [==============================>---------] 79.20% | train_error: 0.495 | train_acc: 0.952 
+
    +
    +
      [==============================>---------] 79.30% | train_error: 0.495 | train_acc: 0.952 
+
    +
    +
      [==============================>---------] 79.40% | train_error: 0.488 | train_acc: 0.953 
+
    +
    +
      [==============================>---------] 79.50% | train_error: 0.482 | train_acc: 0.953 
+
    +
    +
      [==============================>---------] 79.60% | train_error: 0.473 | train_acc: 0.954 
+
    +
    +
      [==============================>---------] 79.70% | train_error: 0.476 | train_acc: 0.954 
+
    +
    +
      [==============================>---------] 79.80% | train_error: 0.477 | train_acc: 0.954 
+
    +
    +
      [==============================>---------] 79.90% | train_error: 0.468 | train_acc: 0.955 
+
    +
    +
      [===============================>--------] 80.00% | train_error: 0.472 | train_acc: 0.954 
+
    +
    +
      [===============================>--------] 80.10% | train_error: 0.466 | train_acc: 0.955 
+
    +
    +
      [===============================>--------] 80.20% | train_error: 0.473 | train_acc: 0.954 
+
    +
    +
      [===============================>--------] 80.30% | train_error: 0.464 | train_acc: 0.955 
+
    +
    +
      [===============================>--------] 80.40% | train_error: 0.467 | train_acc: 0.955 
+
    +
    +
      [===============================>--------] 80.50% | train_error: 0.458 | train_acc: 0.956 
+
    +
    +
      [===============================>--------] 80.60% | train_error: 0.461 | train_acc: 0.955 
+
    +
    +
      [===============================>--------] 80.70% | train_error: 0.449 | train_acc: 0.957 
+
    +
    +
      [===============================>--------] 80.80% | train_error: 0.464 | train_acc: 0.955 
+
    +
    +
      [===============================>--------] 80.90% | train_error: 0.446 | train_acc: 0.957 
+
    +
    +
      [===============================>--------] 81.00% | train_error: 0.456 | train_acc: 0.956 
+
    +
    +
      [===============================>--------] 81.10% | train_error: 0.449 | train_acc: 0.957 
+
    +
    +
      [===============================>--------] 81.20% | train_error: 0.454 | train_acc: 0.956 
+
    +
    +
      [===============================>--------] 81.30% | train_error: 0.446 | train_acc: 0.957 
+
    +
    +
      [===============================>--------] 81.40% | train_error: 0.443 | train_acc: 0.957 
+
    +
    +
      [===============================>--------] 81.50% | train_error: 0.443 | train_acc: 0.957 
+
    +
    +
      [===============================>--------] 81.60% | train_error: 0.429 | train_acc: 0.959 
+
    +
    +
      [===============================>--------] 81.70% | train_error: 0.434 | train_acc: 0.958 
+
    +
    +
      [===============================>--------] 81.80% | train_error: 0.427 | train_acc: 0.959 
+
    +
    +
      [===============================>--------] 81.90% | train_error: 0.422 | train_acc: 0.959 
+
    +
    +
      [===============================>--------] 82.00% | train_error: 0.419 | train_acc: 0.960 
+
    +
    +
      [===============================>--------] 82.10% | train_error: 0.424 | train_acc: 0.959 
+
    +
    +
      [===============================>--------] 82.20% | train_error: 0.424 | train_acc: 0.959 
+
    +
    +
      [===============================>--------] 82.30% | train_error: 0.422 | train_acc: 0.959 
+
    +
    +
      [===============================>--------] 82.40% | train_error: 0.417 | train_acc: 0.960 
+
    +
    +
      [================================>-------] 82.50% | train_error: 0.413 | train_acc: 0.960 
+
    +
    +
      [================================>-------] 82.60% | train_error: 0.408 | train_acc: 0.961 
+
    +
    +
      [================================>-------] 82.70% | train_error: 0.401 | train_acc: 0.961 
+
    +
    +
      [================================>-------] 82.80% | train_error: 0.402 | train_acc: 0.961 
+
    +
    +
      [================================>-------] 82.90% | train_error: 0.396 | train_acc: 0.962 
+
    +
    +
      [================================>-------] 83.00% | train_error: 0.402 | train_acc: 0.961 
+
    +
    +
      [================================>-------] 83.10% | train_error: 0.399 | train_acc: 0.962 
+
    +
    +
      [================================>-------] 83.20% | train_error: 0.401 | train_acc: 0.961 
+
    +
    +
      [================================>-------] 83.30% | train_error: 0.389 | train_acc: 0.962 
+
    +
    +
      [================================>-------] 83.40% | train_error: 0.397 | train_acc: 0.962 
+
    +
    +
      [================================>-------] 83.50% | train_error: 0.386 | train_acc: 0.963 
+
    +
    +
      [================================>-------] 83.60% | train_error: 0.389 | train_acc: 0.963 
+
    +
    +
      [================================>-------] 83.70% | train_error: 0.386 | train_acc: 0.963 
+
    +
    +
      [================================>-------] 83.80% | train_error: 0.385 | train_acc: 0.963 
+
    +
    +
      [================================>-------] 83.90% | train_error: 0.385 | train_acc: 0.963 
+
    +
    +
      [================================>-------] 84.00% | train_error: 0.382 | train_acc: 0.963 
+
    +
    +
      [================================>-------] 84.10% | train_error: 0.378 | train_acc: 0.963 
+
    +
    +
      [================================>-------] 84.20% | train_error: 0.374 | train_acc: 0.964 
+
    +
    +
      [================================>-------] 84.30% | train_error: 0.372 | train_acc: 0.964 
+
    +
    +
      [================================>-------] 84.40% | train_error: 0.371 | train_acc: 0.964 
+
    +
    +
      [================================>-------] 84.50% | train_error: 0.369 | train_acc: 0.964 
+
    +
    +
      [================================>-------] 84.60% | train_error: 0.364 | train_acc: 0.965 
+
    +
    +
      [================================>-------] 84.70% | train_error: 0.372 | train_acc: 0.964 
+
    +
    +
      [================================>-------] 84.80% | train_error: 0.368 | train_acc: 0.964 
+
    +
    +
      [================================>-------] 84.90% | train_error: 0.362 | train_acc: 0.965 
+
    +
    +
      [=================================>------] 85.00% | train_error: 0.364 | train_acc: 0.965 
+
    +
    +
      [=================================>------] 85.10% | train_error: 0.355 | train_acc: 0.966 
+
    +
    +
      [=================================>------] 85.20% | train_error: 0.356 | train_acc: 0.966 
+
    +
    +
      [=================================>------] 85.30% | train_error: 0.349 | train_acc: 0.966 
+
    +
    +
      [=================================>------] 85.40% | train_error: 0.341 | train_acc: 0.967 
+
    +
    +
      [=================================>------] 85.50% | train_error: 0.347 | train_acc: 0.966 
+
    +
    +
      [=================================>------] 85.60% | train_error: 0.349 | train_acc: 0.966 
+
    +
    +
      [=================================>------] 85.70% | train_error: 0.345 | train_acc: 0.967 
+
    +
    +
      [=================================>------] 85.80% | train_error: 0.346 | train_acc: 0.967 
+
    +
    +
      [=================================>------] 85.90% | train_error: 0.338 | train_acc: 0.968 
+
    +
    +
      [=================================>------] 86.00% | train_error: 0.348 | train_acc: 0.966 
+
    +
    +
      [=================================>------] 86.10% | train_error: 0.344 | train_acc: 0.967 
+
    +
    +
      [=================================>------] 86.20% | train_error: 0.346 | train_acc: 0.967 
+
    +
    +
      [=================================>------] 86.30% | train_error: 0.340 | train_acc: 0.967 
+
    +
    +
      [=================================>------] 86.40% | train_error: 0.339 | train_acc: 0.967 
+
    +
    +
      [=================================>------] 86.50% | train_error: 0.336 | train_acc: 0.968 
+
    +
    +
      [=================================>------] 86.60% | train_error: 0.343 | train_acc: 0.967 
+
    +
    +
      [=================================>------] 86.70% | train_error: 0.336 | train_acc: 0.968 
+
    +
    +
      [=================================>------] 86.80% | train_error: 0.339 | train_acc: 0.967 
+
    +
    +
      [=================================>------] 86.90% | train_error: 0.330 | train_acc: 0.968 
+
    +
    +
      [=================================>------] 87.00% | train_error: 0.338 | train_acc: 0.967 
+
    +
    +
      [=================================>------] 87.10% | train_error: 0.328 | train_acc: 0.969 
+
    +
    +
      [=================================>------] 87.20% | train_error: 0.326 | train_acc: 0.969 
+
    +
    +
      [=================================>------] 87.30% | train_error: 0.317 | train_acc: 0.969 
+
    +
    +
      [=================================>------] 87.40% | train_error: 0.329 | train_acc: 0.968 
+
    +
    +
      [==================================>-----] 87.50% | train_error: 0.317 | train_acc: 0.969 
+
    +
    +
      [==================================>-----] 87.60% | train_error: 0.316 | train_acc: 0.970 
+
    +
    +
      [==================================>-----] 87.70% | train_error: 0.318 | train_acc: 0.969 
+
    +
    +
      [==================================>-----] 87.80% | train_error: 0.315 | train_acc: 0.970 
+
    +
    +
      [==================================>-----] 87.90% | train_error: 0.309 | train_acc: 0.970 
+
    +
    +
      [==================================>-----] 88.00% | train_error: 0.308 | train_acc: 0.970 
+
    +
    +
      [==================================>-----] 88.10% | train_error: 0.296 | train_acc: 0.972 
+
    +
    +
      [==================================>-----] 88.20% | train_error: 0.302 | train_acc: 0.971 
+
    +
    +
      [==================================>-----] 88.30% | train_error: 0.298 | train_acc: 0.971 
+
    +
    +
      [==================================>-----] 88.40% | train_error: 0.300 | train_acc: 0.971 
+
    +
    +
      [==================================>-----] 88.50% | train_error: 0.296 | train_acc: 0.971 
+
    +
    +
      [==================================>-----] 88.60% | train_error: 0.291 | train_acc: 0.972 
+
    +
    +
      [==================================>-----] 88.70% | train_error: 0.287 | train_acc: 0.972 
+
    +
    +
      [==================================>-----] 88.80% | train_error: 0.283 | train_acc: 0.973 
+
    +
    +
      [==================================>-----] 88.90% | train_error: 0.280 | train_acc: 0.973 
+
    +
    +
      [==================================>-----] 89.00% | train_error: 0.285 | train_acc: 0.973 
+
    +
    +
      [==================================>-----] 89.10% | train_error: 0.277 | train_acc: 0.973 
+
    +
    +
      [==================================>-----] 89.20% | train_error: 0.292 | train_acc: 0.972 
+
    +
    +
      [==================================>-----] 89.30% | train_error: 0.289 | train_acc: 0.972 
+
    +
    +
      [==================================>-----] 89.40% | train_error: 0.292 | train_acc: 0.972 
+
    +
    +
      [==================================>-----] 89.50% | train_error: 0.287 | train_acc: 0.972 
+
    +
    +
      [==================================>-----] 89.60% | train_error: 0.285 | train_acc: 0.973 
+
    +
    +
      [==================================>-----] 89.70% | train_error: 0.280 | train_acc: 0.973 
+
    +
    +
      [==================================>-----] 89.80% | train_error: 0.283 | train_acc: 0.973 
+
    +
    +
      [==================================>-----] 89.90% | train_error: 0.277 | train_acc: 0.973 
+
    +
    +
      [===================================>----] 90.00% | train_error: 0.285 | train_acc: 0.973 
+
    +
    +
      [===================================>----] 90.10% | train_error: 0.278 | train_acc: 0.973 
+
    +
    +
      [===================================>----] 90.20% | train_error: 0.266 | train_acc: 0.974 
+
    +
    +
      [===================================>----] 90.30% | train_error: 0.264 | train_acc: 0.975 
+
    +
    +
      [===================================>----] 90.40% | train_error: 0.271 | train_acc: 0.974 
+
    +
    +
      [===================================>----] 90.50% | train_error: 0.265 | train_acc: 0.974 
+
    +
    +
      [===================================>----] 90.60% | train_error: 0.265 | train_acc: 0.974 
+
    +
    +
      [===================================>----] 90.70% | train_error: 0.258 | train_acc: 0.975 
+
    +
    +
      [===================================>----] 90.80% | train_error: 0.263 | train_acc: 0.975 
+
    +
    +
      [===================================>----] 90.90% | train_error: 0.251 | train_acc: 0.976 
+
    +
    +
      [===================================>----] 91.00% | train_error: 0.248 | train_acc: 0.976 
+
    +
    +
      [===================================>----] 91.10% | train_error: 0.248 | train_acc: 0.976 
+
    +
    +
      [===================================>----] 91.20% | train_error: 0.250 | train_acc: 0.976 
+
    +
    +
      [===================================>----] 91.30% | train_error: 0.243 | train_acc: 0.977 
+
    +
    +
      [===================================>----] 91.40% | train_error: 0.241 | train_acc: 0.977 
+
    +
    +
      [===================================>----] 91.50% | train_error: 0.239 | train_acc: 0.977 
+
    +
    +
      [===================================>----] 91.60% | train_error: 0.240 | train_acc: 0.977 
+
    +
    +
      [===================================>----] 91.70% | train_error: 0.239 | train_acc: 0.977 
+
    +
    +
      [===================================>----] 91.80% | train_error: 0.239 | train_acc: 0.977 
+
    +
    +
      [===================================>----] 91.90% | train_error: 0.235 | train_acc: 0.977 
+
    +
    +
      [===================================>----] 92.00% | train_error: 0.233 | train_acc: 0.978 
+
    +
    +
      [===================================>----] 92.10% | train_error: 0.234 | train_acc: 0.977 
+
    +
    +
      [===================================>----] 92.20% | train_error: 0.240 | train_acc: 0.977 
+
    +
    +
      [===================================>----] 92.30% | train_error: 0.238 | train_acc: 0.977 
+
    +
    +
      [===================================>----] 92.40% | train_error: 0.226 | train_acc: 0.978 
+
    +
    +
      [====================================>---] 92.50% | train_error: 0.226 | train_acc: 0.978 
+
    +
    +
      [====================================>---] 92.60% | train_error: 0.229 | train_acc: 0.978 
+
    +
    +
      [====================================>---] 92.70% | train_error: 0.226 | train_acc: 0.978 
+
    +
    +
      [====================================>---] 92.80% | train_error: 0.218 | train_acc: 0.979 
+
    +
    +
      [====================================>---] 92.90% | train_error: 0.219 | train_acc: 0.979 
+
    +
    +
      [====================================>---] 93.00% | train_error: 0.220 | train_acc: 0.979 
+
    +
    +
      [====================================>---] 93.10% | train_error: 0.216 | train_acc: 0.979 
+
    +
    +
      [====================================>---] 93.20% | train_error: 0.217 | train_acc: 0.979 
+
    +
    +
      [====================================>---] 93.30% | train_error: 0.216 | train_acc: 0.979 
+
    +
    +
      [====================================>---] 93.40% | train_error: 0.216 | train_acc: 0.979 
+
    +
    +
      [====================================>---] 93.50% | train_error: 0.213 | train_acc: 0.979 
+
    +
    +
      [====================================>---] 93.60% | train_error: 0.213 | train_acc: 0.979 
+
    +
    +
      [====================================>---] 93.70% | train_error: 0.213 | train_acc: 0.979 
+
    +
    +
      [====================================>---] 93.80% | train_error: 0.205 | train_acc: 0.980 
+
    +
    +
      [====================================>---] 93.90% | train_error: 0.210 | train_acc: 0.980 
+
    +
    +
      [====================================>---] 94.00% | train_error: 0.211 | train_acc: 0.980 
+
    +
    +
      [====================================>---] 94.10% | train_error: 0.209 | train_acc: 0.980 
+
    +
    +
      [====================================>---] 94.20% | train_error: 0.206 | train_acc: 0.980 
+
    +
    +
      [====================================>---] 94.30% | train_error: 0.204 | train_acc: 0.980 
+
    +
    +
      [====================================>---] 94.40% | train_error: 0.210 | train_acc: 0.980 
+
    +
    +
      [====================================>---] 94.50% | train_error: 0.198 | train_acc: 0.981 
+
    +
    +
      [====================================>---] 94.60% | train_error: 0.198 | train_acc: 0.981 
+
    +
    +
      [====================================>---] 94.70% | train_error: 0.202 | train_acc: 0.981 
+
    +
    +
      [====================================>---] 94.80% | train_error: 0.201 | train_acc: 0.981 
+
    +
    +
      [====================================>---] 94.90% | train_error: 0.202 | train_acc: 0.981 
+
    +
    +
      [=====================================>--] 95.00% | train_error: 0.195 | train_acc: 0.981 
+
    +
    +
      [=====================================>--] 95.10% | train_error: 0.203 | train_acc: 0.980 
+
    +
    +
      [=====================================>--] 95.20% | train_error: 0.197 | train_acc: 0.981 
+
    +
    +
      [=====================================>--] 95.30% | train_error: 0.201 | train_acc: 0.981 
+
    +
    +
      [=====================================>--] 95.40% | train_error: 0.187 | train_acc: 0.982 
+
    +
    +
      [=====================================>--] 95.50% | train_error: 0.197 | train_acc: 0.981 
+
    +
    +
      [=====================================>--] 95.60% | train_error: 0.187 | train_acc: 0.982 
+
    +
    +
      [=====================================>--] 95.70% | train_error: 0.195 | train_acc: 0.981 
+
    +
    +
      [=====================================>--] 95.80% | train_error: 0.195 | train_acc: 0.981 
+
    +
    +
      [=====================================>--] 95.90% | train_error: 0.190 | train_acc: 0.982 
+
    +
    +
      [=====================================>--] 96.00% | train_error: 0.190 | train_acc: 0.982 
+
    +
    +
      [=====================================>--] 96.10% | train_error: 0.194 | train_acc: 0.981 
+
    +
    +
      [=====================================>--] 96.20% | train_error: 0.187 | train_acc: 0.982 
+
    +
    +
      [=====================================>--] 96.30% | train_error: 0.190 | train_acc: 0.982 
+
    +
    +
      [=====================================>--] 96.40% | train_error: 0.190 | train_acc: 0.982 
+
    +
    +
      [=====================================>--] 96.50% | train_error: 0.195 | train_acc: 0.981 
+
    +
    +
      [=====================================>--] 96.60% | train_error: 0.188 | train_acc: 0.982 
+
    +
    +
      [=====================================>--] 96.70% | train_error: 0.188 | train_acc: 0.982 
+
    +
    +
      [=====================================>--] 96.80% | train_error: 0.190 | train_acc: 0.982 
+
    +
    +
      [=====================================>--] 96.90% | train_error: 0.191 | train_acc: 0.982 
+
    +
    +
      [=====================================>--] 97.00% | train_error: 0.188 | train_acc: 0.982 
+
    +
    +
      [=====================================>--] 97.10% | train_error: 0.189 | train_acc: 0.982 
+
    +
    +
      [=====================================>--] 97.20% | train_error: 0.175 | train_acc: 0.983 
+
    +
    +
      [=====================================>--] 97.30% | train_error: 0.189 | train_acc: 0.982 
+
    +
    +
      [=====================================>--] 97.40% | train_error: 0.183 | train_acc: 0.982 
+
    +
    +
      [======================================>-] 97.50% | train_error: 0.189 | train_acc: 0.982 
+
    +
    +
      [======================================>-] 97.60% | train_error: 0.181 | train_acc: 0.983 
+
    +
    +
      [======================================>-] 97.70% | train_error: 0.188 | train_acc: 0.982 
+
    +
    +
      [======================================>-] 97.80% | train_error: 0.179 | train_acc: 0.983 
+
    +
    +
      [======================================>-] 97.90% | train_error: 0.190 | train_acc: 0.982 
+
    +
    +
      [======================================>-] 98.00% | train_error: 0.186 | train_acc: 0.982 
+
    +
    +
      [======================================>-] 98.10% | train_error: 0.185 | train_acc: 0.982 
+
    +
    +
      [======================================>-] 98.20% | train_error: 0.182 | train_acc: 0.982 
+
    +
    +
      [======================================>-] 98.30% | train_error: 0.182 | train_acc: 0.982 
+
    +
    +
      [======================================>-] 98.40% | train_error: 0.171 | train_acc: 0.984 
+
    +
    +
      [======================================>-] 98.50% | train_error: 0.178 | train_acc: 0.983 
+
    +
    +
      [======================================>-] 98.60% | train_error: 0.170 | train_acc: 0.984 
+
    +
    +
      [======================================>-] 98.70% | train_error: 0.181 | train_acc: 0.983 
+
    +
    +
      [======================================>-] 98.80% | train_error: 0.166 | train_acc: 0.984 
+
    +
    +
      [======================================>-] 98.90% | train_error: 0.175 | train_acc: 0.983 
+
    +
    +
      [======================================>-] 99.00% | train_error: 0.173 | train_acc: 0.983 
+
    +
    +
      [======================================>-] 99.10% | train_error: 0.173 | train_acc: 0.983 
+
    +
    +
      [======================================>-] 99.20% | train_error: 0.171 | train_acc: 0.984 
+
    +
    +
      [======================================>-] 99.30% | train_error: 0.168 | train_acc: 0.984 
+
    +
    +
      [======================================>-] 99.40% | train_error: 0.167 | train_acc: 0.984 
+
    +
    +
      [======================================>-] 99.50% | train_error: 0.174 | train_acc: 0.983 
+
    +
    +
      [======================================>-] 99.60% | train_error: 0.158 | train_acc: 0.985 
+
    +
    +
      [======================================>-] 99.70% | train_error: 0.173 | train_acc: 0.983 
+
    +
    +
      [======================================>-] 99.80% | train_error: 0.164 | train_acc: 0.984 
+
    +
    +
      [======================================>-] 99.90% | train_error: 0.175 | train_acc: 0.983 
+
    +
    +
                                                                                                
+
    +
    +
      [=======================================>] 100.0% | train_error: 0.175 | train_acc: 0.983 
+
    +
    +
    +
    +
    +