lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

small update on neural networks

Browse Source
This commit is contained in:
mhjensen
2018-10-16 13:54:54 +02:00
parent ce0c90eccd
commit 9bdb28895e
75 changed files with 6318 additions and 5703 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/pub/NeuralNet/html/._NeuralNet-bs049.html
+91 -87
View File
@@ -92,54 +92,55 @@ Automatically generated HTML file from DocOnce source
'___sec30'),
('Defining the cost function', 2, None, '___sec31'),
('Example: binary classification problem', 2, None, '___sec32'),
('The Softmax function', 2, None, '___sec33'),
('Developing a code for doing neural networks with back '
'propagation',
2,
None,
'___sec33'),
('Collect and pre-process data', 2, None, '___sec34'),
('Train and test datasets', 2, None, '___sec35'),
('Define model and architecture', 2, None, '___sec36'),
('Layers', 2, None, '___sec37'),
('Weights and biases', 2, None, '___sec38'),
('Feed-forward pass', 2, None, '___sec39'),
('Matrix multiplications', 2, None, '___sec40'),
('Choose cost function and optimizer', 2, None, '___sec41'),
('Optimizing the cost function', 2, None, '___sec42'),
('Regularization', 2, None, '___sec43'),
('Matrix multiplication', 2, None, '___sec44'),
('Improving performance', 2, None, '___sec45'),
('Full object-oriented implementation', 2, None, '___sec46'),
('Evaluate model performance on test data', 2, None, '___sec47'),
('Adjust hyperparameters', 2, None, '___sec48'),
('Visualization', 2, None, '___sec49'),
('scikit-learn implementation', 2, None, '___sec50'),
('Visualization', 2, None, '___sec51'),
'___sec34'),
('Collect and pre-process data', 2, None, '___sec35'),
('Train and test datasets', 2, None, '___sec36'),
('Define model and architecture', 2, None, '___sec37'),
('Layers', 2, None, '___sec38'),
('Weights and biases', 2, None, '___sec39'),
('Feed-forward pass', 2, None, '___sec40'),
('Matrix multiplications', 2, None, '___sec41'),
('Choose cost function and optimizer', 2, None, '___sec42'),
('Optimizing the cost function', 2, None, '___sec43'),
('Regularization', 2, None, '___sec44'),
('Matrix multiplication', 2, None, '___sec45'),
('Improving performance', 2, None, '___sec46'),
('Full object-oriented implementation', 2, None, '___sec47'),
('Evaluate model performance on test data', 2, None, '___sec48'),
('Adjust hyperparameters', 2, None, '___sec49'),
('Visualization', 2, None, '___sec50'),
('scikit-learn implementation', 2, None, '___sec51'),
('Visualization', 2, None, '___sec52'),
('Building neural networks in Tensorflow and Keras',
2,
None,
'___sec52'),
('Tensorflow', 2, None, '___sec53'),
('Collect and pre-process data', 2, None, '___sec54'),
('Using TensorFlow backend', 2, None, '___sec55'),
('Optimizing and using gradient descent', 2, None, '___sec56'),
('Using Keras', 2, None, '___sec57'),
('Which activation function should I use?', 2, None, '___sec58'),
'___sec53'),
('Tensorflow', 2, None, '___sec54'),
('Collect and pre-process data', 2, None, '___sec55'),
('Using TensorFlow backend', 2, None, '___sec56'),
('Optimizing and using gradient descent', 2, None, '___sec57'),
('Using Keras', 2, None, '___sec58'),
('Which activation function should I use?', 2, None, '___sec59'),
('Is the Logistic activation function (Sigmoid) our choice?',
2,
None,
'___sec59'),
('The derivative of the Logistic funtion', 2, None, '___sec60'),
('The RELU function family', 2, None, '___sec61'),
('Which activation function should we use?', 2, None, '___sec62'),
'___sec60'),
('The derivative of the Logistic funtion', 2, None, '___sec61'),
('The RELU function family', 2, None, '___sec62'),
('Which activation function should we use?', 2, None, '___sec63'),
('A top-down perspective on Neural networks',
2,
None,
'___sec63'),
'___sec64'),
('Limitations of supervised learning with deep networks',
2,
None,
'___sec64')]}
'___sec65')]}
end of tocinfo -->
<body>
@@ -210,38 +211,39 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Layers</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Weights and biases</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Regularization</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Improving performance</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
<!-- navigation toc: --> <li><a href="#___sec48" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Visualization</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Visualization</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Tensorflow</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Using Keras</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The RELU function family</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
<!-- navigation toc: --> <li><a href="#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
</ul>
</li>
@@ -257,35 +259,37 @@ MathJax.Hub.Config({
<a name="part0049"></a>
<!-- !split -->
<h2 id="___sec48" class="anchor">Adjust hyperparameters </h2>
<h2 id="___sec48" class="anchor">Evaluate model performance on test data </h2>
<p>
We now perform a grid search to find the optimal hyperparameters for the network.
Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around \( 98\% \) (\( 2\% \) error rate).
To measure the performance of our network we evaluate how well it does it data it has never seen before, i.e. the test data.
We measure the performance of the network using the <em>accuracy</em> score.
The accuracy is as you would expect just the number of images correctly labeled divided by the total number of images. A perfect classifier will have an accuracy score of \( 1 \).
$$ \text{Accuracy} = \frac{\sum_{i=1}^n I(\hat{y}_i = y_i)}{n} ,$$
<p>
where \( I \) is the indicator function, \( 1 \) if \( \hat{y}_i = y_i \) and \( 0 \) otherwise.
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>eta_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
lmbd_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
<span style="color: #408080; font-style: italic"># store the models for later use</span>
DNN_numpy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)), dtype<span style="color: #666666">=</span><span style="color: #008000">object</span>)
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>epochs <span style="color: #666666">=</span> <span style="color: #666666">100</span>
batch_size <span style="color: #666666">=</span> <span style="color: #666666">100</span>
<span style="color: #408080; font-style: italic"># grid search</span>
<span style="color: #008000; font-weight: bold">for</span> i, eta <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(eta_vals):
<span style="color: #008000; font-weight: bold">for</span> j, lmbd <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(lmbd_vals):
dnn <span style="color: #666666">=</span> NeuralNetwork(X_train, Y_train_onehot, eta<span style="color: #666666">=</span>eta, lmbd<span style="color: #666666">=</span>lmbd, epochs<span style="color: #666666">=</span>epochs, batch_size<span style="color: #666666">=</span>batch_size,
n_hidden_neurons<span style="color: #666666">=</span>n_hidden_neurons, n_categories<span style="color: #666666">=</span>n_categories)
dnn<span style="color: #666666">.</span>train()
DNN_numpy[i][j] <span style="color: #666666">=</span> dnn
test_predict <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_test)
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">&quot;Learning rate = &quot;</span>, eta)
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">&quot;Lambda = &quot;</span>, lmbd)
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">&quot;Accuracy score on test set: &quot;</span>, accuracy_score(Y_test, test_predict))
<span style="color: #008000; font-weight: bold">print</span>()
dnn <span style="color: #666666">=</span> NeuralNetwork(X_train, Y_train_onehot, eta<span style="color: #666666">=</span>eta, lmbd<span style="color: #666666">=</span>lmbd, epochs<span style="color: #666666">=</span>epochs, batch_size<span style="color: #666666">=</span>batch_size,
n_hidden_neurons<span style="color: #666666">=</span>n_hidden_neurons, n_categories<span style="color: #666666">=</span>n_categories)
dnn<span style="color: #666666">.</span>train()
test_predict <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_test)
<span style="color: #408080; font-style: italic"># accuracy score from scikit library</span>
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">&quot;Accuracy score on test set: &quot;</span>, accuracy_score(Y_test, test_predict))
<span style="color: #408080; font-style: italic"># equivalent in numpy</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">accuracy_score_numpy</span>(Y_test, Y_pred):
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>sum(Y_test <span style="color: #666666">==</span> Y_pred) <span style="color: #666666">/</span> <span style="color: #008000">len</span>(Y_test)
<span style="color: #408080; font-style: italic">#print(&quot;Accuracy score on test set: &quot;, accuracy_score_numpy(Y_test, test_predict))</span>
</pre></div>
<p>
<p>
@@ -313,7 +317,7 @@ DNN_numpy <span style="color: #666666">=</span> np<span style="color: #666666">.
<li><a href="._NeuralNet-bs057.html">58</a></li>
<li><a href="._NeuralNet-bs058.html">59</a></li>
<li><a href="">...</a></li>
<li><a href="._NeuralNet-bs065.html">66</a></li>
<li><a href="._NeuralNet-bs066.html">67</a></li>
<li><a href="._NeuralNet-bs050.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->