updated typos
This commit is contained in:
mhjensen
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -250,7 +251,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Oct 2, 2018</h4></center> <!-- date -->
|
||||
<center><h4>Oct 4, 2018</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
|
||||
@@ -274,7 +275,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._NeuralNet-bs008.html">9</a></li>
|
||||
<li><a href="._NeuralNet-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -259,7 +260,7 @@ a weight variable.
|
||||
<li><a href="._NeuralNet-bs009.html">10</a></li>
|
||||
<li><a href="._NeuralNet-bs010.html">11</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs002.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -309,7 +310,7 @@ humanities to life science and medicine.
|
||||
<li><a href="._NeuralNet-bs010.html">11</a></li>
|
||||
<li><a href="._NeuralNet-bs011.html">12</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs003.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -274,7 +275,7 @@ methods we discussed earlier.
|
||||
<li><a href="._NeuralNet-bs011.html">12</a></li>
|
||||
<li><a href="._NeuralNet-bs012.html">13</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs004.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -266,7 +267,7 @@ to <em>all</em> nodes in the subsequent layer, making this a so-called
|
||||
<li><a href="._NeuralNet-bs012.html">13</a></li>
|
||||
<li><a href="._NeuralNet-bs013.html">14</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs005.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -275,7 +276,7 @@ recognition.
|
||||
<li><a href="._NeuralNet-bs013.html">14</a></li>
|
||||
<li><a href="._NeuralNet-bs014.html">15</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs006.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -267,7 +268,7 @@ especially well-suited for handwriting and speech recognition.
|
||||
<li><a href="._NeuralNet-bs014.html">15</a></li>
|
||||
<li><a href="._NeuralNet-bs015.html">16</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs007.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -268,7 +269,7 @@ type of NN due the unusual activation functions.
|
||||
<li><a href="._NeuralNet-bs015.html">16</a></li>
|
||||
<li><a href="._NeuralNet-bs016.html">17</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs008.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -266,7 +267,7 @@ Such networks are often called <em>multilayer perceptrons</em> (MLPs).
|
||||
<li><a href="._NeuralNet-bs016.html">17</a></li>
|
||||
<li><a href="._NeuralNet-bs017.html">18</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs009.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -271,7 +272,7 @@ as to not restrict the range of output values.
|
||||
<li><a href="._NeuralNet-bs017.html">18</a></li>
|
||||
<li><a href="._NeuralNet-bs018.html">19</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs010.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -240,7 +241,7 @@ $$
|
||||
$$
|
||||
|
||||
This function receives \( x_i \) as inputs.
|
||||
Here the activation \( z=\sum_{i=1}^n w_ix_i \).
|
||||
Here the activation \( z=(\sum_{i=1}^n w_ix_i+b_i) \).
|
||||
In an FFNN of such neurons, the <em>inputs</em> \( x_i \) are the <em>outputs</em> of
|
||||
the neurons in the preceding layer. Furthermore, an MLP is
|
||||
fully-connected, which means that each neuron receives a weighted sum
|
||||
@@ -272,7 +273,7 @@ of the outputs of <em>all</em> neurons in the previous layer.
|
||||
<li><a href="._NeuralNet-bs018.html">19</a></li>
|
||||
<li><a href="._NeuralNet-bs019.html">20</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs011.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -301,7 +302,7 @@ is obtained.
|
||||
<li><a href="._NeuralNet-bs019.html">20</a></li>
|
||||
<li><a href="._NeuralNet-bs020.html">21</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs012.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -283,7 +284,7 @@ $$
|
||||
<li><a href="._NeuralNet-bs020.html">21</a></li>
|
||||
<li><a href="._NeuralNet-bs021.html">22</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs013.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -274,7 +275,7 @@ variables are the input values \( x_n \).
|
||||
<li><a href="._NeuralNet-bs021.html">22</a></li>
|
||||
<li><a href="._NeuralNet-bs022.html">23</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs014.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -283,7 +284,7 @@ flexibility of a neural network.
|
||||
<li><a href="._NeuralNet-bs022.html">23</a></li>
|
||||
<li><a href="._NeuralNet-bs023.html">24</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs015.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -244,7 +245,7 @@ is the bias \( b_i^l \) and activation \( y_i^l \) of node \( i \) in layer \( l
|
||||
<p>
|
||||
We have that \( \mathrm{W}_l \) is an \( N_{l-1} \times N_l \) matrix, while \( \hat{b}_l \) and \( \hat{y}_l \) are \( N_l \times 1 \) column vectors.
|
||||
With this notation, the sum becomes a matrix-vector multiplication, and we can write
|
||||
the equation for the activations of hidden layer 2 as
|
||||
the equation for the activations of hidden layer 2 (assuming three nodes for simplicity) as
|
||||
$$
|
||||
\begin{equation}
|
||||
\hat{y}_2 = f_2(\mathrm{W}_2 \hat{y}_{1} + \hat{b}_{2}) =
|
||||
@@ -293,7 +294,7 @@ $$
|
||||
<li><a href="._NeuralNet-bs023.html">24</a></li>
|
||||
<li><a href="._NeuralNet-bs024.html">25</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs016.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -277,7 +278,7 @@ used as input to the activation functions. For each operation
|
||||
<li><a href="._NeuralNet-bs024.html">25</a></li>
|
||||
<li><a href="._NeuralNet-bs025.html">26</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs017.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -271,7 +272,7 @@ for a FFNN to fulfill the universal approximation theorem
|
||||
<li><a href="._NeuralNet-bs025.html">26</a></li>
|
||||
<li><a href="._NeuralNet-bs026.html">27</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs018.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -279,7 +280,7 @@ $$
|
||||
<li><a href="._NeuralNet-bs026.html">27</a></li>
|
||||
<li><a href="._NeuralNet-bs027.html">28</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs019.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -341,7 +342,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._NeuralNet-bs027.html">28</a></li>
|
||||
<li><a href="._NeuralNet-bs028.html">29</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs020.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -246,7 +247,7 @@ As a convention it is normal to call a network with one layer of input units,
|
||||
units and one layer of output units as a two-layer network. A network with two layers of hidden units is called a three-layer network etc etc.
|
||||
|
||||
<p>
|
||||
For an MLP there is no direct connection between the output nodes/neurons/units and the input nodes/neurons/units.
|
||||
For an MLP network there is no direct connection between the output nodes/neurons/units and the input nodes/neurons/units.
|
||||
Hereafter we will call the various entities of a layer for nodes.
|
||||
There are also no connections within a single layer.
|
||||
|
||||
@@ -292,7 +293,7 @@ like logistic regression or linear regression and their modifications on the oth
|
||||
<li><a href="._NeuralNet-bs028.html">29</a></li>
|
||||
<li><a href="._NeuralNet-bs029.html">30</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs021.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -282,7 +283,7 @@ the potential of being universal approximators.
|
||||
<li><a href="._NeuralNet-bs029.html">30</a></li>
|
||||
<li><a href="._NeuralNet-bs030.html">31</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs022.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -288,7 +289,7 @@ classes.
|
||||
<li><a href="._NeuralNet-bs030.html">31</a></li>
|
||||
<li><a href="._NeuralNet-bs031.html">32</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs023.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -256,11 +257,10 @@ $$
|
||||
$$
|
||||
|
||||
<p>
|
||||
With the activation function \( \hat{z}^l \) we can in turn define the
|
||||
With the activation values \( \hat{z}^l \) we can in turn define the
|
||||
output of layer \( l \) as \( \hat{a}^l = f(\hat{z}^l) \) where \( f \) is our
|
||||
activation function. In the examples here we will use the sigmoid
|
||||
function discussed in our logistic regression lectures and here as
|
||||
well. We will also use the same activation function \( f \) for all layers
|
||||
function discussed in our logistic regression lectures. We will also use the same activation function \( f \) for all layers
|
||||
and their nodes. It means we have
|
||||
|
||||
$$
|
||||
@@ -293,7 +293,7 @@ $$
|
||||
<li><a href="._NeuralNet-bs031.html">32</a></li>
|
||||
<li><a href="._NeuralNet-bs032.html">33</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs024.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -276,7 +277,7 @@ $$
|
||||
<li><a href="._NeuralNet-bs032.html">33</a></li>
|
||||
<li><a href="._NeuralNet-bs033.html">34</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs025.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -279,7 +280,7 @@ $$
|
||||
<li><a href="._NeuralNet-bs033.html">34</a></li>
|
||||
<li><a href="._NeuralNet-bs034.html">35</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs026.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -252,7 +253,7 @@ $$
|
||||
|
||||
<p>
|
||||
This is an important expression. The second term on the right handside
|
||||
measures how fast the cost is changing as a function of the $j$th
|
||||
measures how fast the cost function is changing as a function of the $j$th
|
||||
output activation. If, for example, the cost function doesn't depend
|
||||
much on a particular output node \( j \), then \( \delta_j^L \) will be small,
|
||||
which is what we would expect. The first term on the right, measures
|
||||
@@ -264,9 +265,9 @@ Notice that everything in the above equations is easily computed. In
|
||||
particular, we compute \( z_j^L \) while computing the behaviour of the
|
||||
network, and it is only a small additional overhead to compute
|
||||
\( f'(z^L_j) \). The exact form of the derivative with respect to the
|
||||
outpuwill, of course, depend on the form of the cost function.
|
||||
output depends on the form of the cost function.
|
||||
However, provided the cost function is known there should be little
|
||||
trouble computing
|
||||
trouble in calculating
|
||||
|
||||
$$
|
||||
\frac{\partial {\cal C}}{\partial (a_j^L)}
|
||||
@@ -304,7 +305,7 @@ $$
|
||||
<li><a href="._NeuralNet-bs034.html">35</a></li>
|
||||
<li><a href="._NeuralNet-bs035.html">36</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs027.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -271,7 +272,7 @@ That is, the error \( \delta_j^L \) is exactly equal to the rate of change of th
|
||||
<li><a href="._NeuralNet-bs035.html">36</a></li>
|
||||
<li><a href="._NeuralNet-bs036.html">37</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs028.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -235,6 +236,8 @@ MathJax.Hub.Config({
|
||||
|
||||
<p>
|
||||
We have now three equations that are essential for the computations of the derivatives of the cost function at the output layer. These equations are needed to start the algorithm and they are
|
||||
|
||||
<p>
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-body">
|
||||
<p> <!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
|
||||
@@ -276,7 +279,7 @@ descent. In this case we say the system learns slowly.
|
||||
|
||||
<p>
|
||||
Another interesting feature is that is when the activation function,
|
||||
represented by the sigmoid function here, is rather flat when towards
|
||||
represented by the sigmoid function here, is rather flat when we move towards
|
||||
its end values \( 0 \) and \( 1 \) (see the above Python codes). In these
|
||||
cases, the derivatives of the activation function will also be close
|
||||
to zero, meaning again that the gradients will be small and the
|
||||
@@ -314,7 +317,7 @@ one \( L-1 \) in terms of the errors in the final output layer.
|
||||
<li><a href="._NeuralNet-bs036.html">37</a></li>
|
||||
<li><a href="._NeuralNet-bs037.html">38</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs029.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -286,7 +287,7 @@ We are now ready to set up the algorithm for back propagation and learning the w
|
||||
<li><a href="._NeuralNet-bs037.html">38</a></li>
|
||||
<li><a href="._NeuralNet-bs038.html">39</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs030.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -302,7 +303,7 @@ $$
|
||||
|
||||
<p>
|
||||
The parameter \( \eta \) is the learning parameter discussed in connection with the gradient descent methods.
|
||||
Here it is convenient to use stochastic radient descent with mini-batches with an outer loop that steps through multiple epochs of training.
|
||||
Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -330,7 +331,7 @@ Here it is convenient to use stochastic radient descent with mini-batches with a
|
||||
<li><a href="._NeuralNet-bs038.html">39</a></li>
|
||||
<li><a href="._NeuralNet-bs039.html">40</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs031.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -251,10 +252,10 @@ calculate.
|
||||
|
||||
<p>
|
||||
For an input \( \boldsymbol{a} \) from the hidden layer, the probability that the input \( \boldsymbol{x} \)
|
||||
is in class 0 or 1 is just:
|
||||
|
||||
is in class 0 or 1 is just. We let \( \theta \) represent the unknown weights and biases to be adjusted by our equations). The variable \( x \)
|
||||
represents our activation values \( z \). We have
|
||||
$$
|
||||
P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) = \frac{1}{1 + \exp (- \boldsymbol{a}^T \boldsymbol{w}_{out})} ,
|
||||
P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) = \frac{1}{1 + \exp (- \boldsymbol{x}} ,
|
||||
$$
|
||||
|
||||
and
|
||||
@@ -292,7 +293,7 @@ of our network.
|
||||
<li><a href="._NeuralNet-bs039.html">40</a></li>
|
||||
<li><a href="._NeuralNet-bs040.html">41</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs032.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -315,7 +316,7 @@ We leave it as an exercise in project 2 to derive these equations.
|
||||
<li><a href="._NeuralNet-bs040.html">41</a></li>
|
||||
<li><a href="._NeuralNet-bs041.html">42</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs033.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -270,7 +271,7 @@ One can identify a set of key steps when using neural networks to solve supervis
|
||||
<li><a href="._NeuralNet-bs041.html">42</a></li>
|
||||
<li><a href="._NeuralNet-bs042.html">43</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs034.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -349,7 +350,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._NeuralNet-bs042.html">43</a></li>
|
||||
<li><a href="._NeuralNet-bs043.html">44</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs035.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -303,7 +304,7 @@ X_train, X_test, Y_train, Y_test <span style="color: #666666">=</span> train_tes
|
||||
<li><a href="._NeuralNet-bs043.html">44</a></li>
|
||||
<li><a href="._NeuralNet-bs044.html">45</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs036.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -300,7 +301,7 @@ which is inspired by probability theory (see logistic regression) and was most c
|
||||
<li><a href="._NeuralNet-bs044.html">45</a></li>
|
||||
<li><a href="._NeuralNet-bs045.html">46</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs037.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -267,7 +268,7 @@ i.e. each neuron \( j \) outputs the probability of being in class \( j \) given
|
||||
The denominator is a normalization factor to ensure the outputs (probabilities) sum up to 1.
|
||||
The exponent is just the weighted sum of inputs as before:
|
||||
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i = \boldsymbol{a}^T \boldsymbol{w}_j .$$
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i+b_j.$$
|
||||
|
||||
<p>
|
||||
Since each neuron in the output layer is connected to the 50 inputs from the hidden layer we have 50x10 = 500
|
||||
@@ -299,7 +300,7 @@ weights to the output layer.
|
||||
<li><a href="._NeuralNet-bs045.html">46</a></li>
|
||||
<li><a href="._NeuralNet-bs046.html">47</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs038.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -241,7 +242,7 @@ or normal distribution. Setting all weights to zero means all neurons give the s
|
||||
Adding a bias value to the weighted sum of inputs allows the neural network to represent a greater range
|
||||
of values. Without it, any input with the value 0 will be mapped to zero (before being passed through the activation). The bias unit has an output of 1, and a weight to each neuron \( j \), \( b_j \):
|
||||
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i + 1\cdot b_j = \boldsymbol{a}^T \boldsymbol{w}_j + b_j .$$
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i + 1\cdot b_j.$$
|
||||
|
||||
<p>
|
||||
The bias weights \( \boldsymbol{b} \) are often initialized to zero, but a small value like \( 0.01 \) ensures all neurons have some output which can be backpropagated in the first training cycle.
|
||||
@@ -290,7 +291,7 @@ output_bias <span style="color: #666666">=</span> np<span style="color: #666666"
|
||||
<li><a href="._NeuralNet-bs046.html">47</a></li>
|
||||
<li><a href="._NeuralNet-bs047.html">48</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs039.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -235,25 +236,25 @@ MathJax.Hub.Config({
|
||||
|
||||
<p>
|
||||
Denote \( F \) the number of features, \( H \) the number of hidden neurons and \( C \) the number of categories.
|
||||
For each input image we calculate a weighted sum of input features (pixel values) to each neuron \( j \) in the hidden layer:
|
||||
For each input image we calculate a weighted sum of input features (pixel values) to each neuron \( j \) in the hidden layer \( l \):
|
||||
|
||||
$$ z_{j}^{h} = \sum_{i=1}^{F} w_{ij}^{h} x_i + b_{j}^{h} = \boldsymbol{x}^T \boldsymbol{w}_{j}^{h} + b_{j}^{h} ,$$
|
||||
$$ z_{j}^{l} = \sum_{i=1}^{F} w_{ij}^{l} x_i + b_{j}^{l},$$
|
||||
|
||||
<p>
|
||||
this is then passed through our activation function
|
||||
|
||||
$$ a_{j}^{h} = f(z_{j}^{h}) .$$
|
||||
$$ a_{j}^{l} = f(z_{j}^{l}) .$$
|
||||
|
||||
<p>
|
||||
We calculate a weighted sum of inputs (activations in the hidden layer) to each neuron \( j \) in the output layer:
|
||||
|
||||
$$ z_{j}^{o} = \sum_{i=1}^{H} w_{ij}^{o} a_{i}^{h} + b_{j}^{o} = (\boldsymbol{a}^{h})^T \boldsymbol{w}_{j}^{o} + b_{j}^{o} .$$
|
||||
$$ z_{j}^{L} = \sum_{i=1}^{H} w_{ij}^{L} a_{i}^{l} + b_{j}^{L}.$$
|
||||
|
||||
<p>
|
||||
Finally we calculate the output of neuron \( j \) in the output layer using the softmax function:
|
||||
|
||||
$$ a_{j}^{o} = \frac{\exp{(z_j^{o})}}
|
||||
{\sum_{c=0}^{C-1} \exp{(z_c^{o})}} .$$
|
||||
$$ a_{j}^{L} = \frac{\exp{(z_j^{L})}}
|
||||
{\sum_{c=0}^{C-1} \exp{(z_c^{L})}} .$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -281,7 +282,7 @@ $$ a_{j}^{o} = \frac{\exp{(z_j^{o})}}
|
||||
<li><a href="._NeuralNet-bs047.html">48</a></li>
|
||||
<li><a href="._NeuralNet-bs048.html">49</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs040.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -246,28 +247,28 @@ and obtain a matrix that holds the weighted sum of inputs to the hidden layer
|
||||
for each input image and each hidden neuron.
|
||||
We also add the bias to obtain a matrix of weighted sums to the hidden layer \( Z^{h} \):
|
||||
|
||||
$$ Z^{h} = X W^{h} + B^{h} ,$$
|
||||
$$ \hat{z}^{l} = \hat{X} \hat{W}^{l} + \hat{b}^{l} ,$$
|
||||
|
||||
<p>
|
||||
meaning the same bias (1D array with size equal number of hidden neurons) is added to each input image.
|
||||
This is then passed through the activation:
|
||||
|
||||
$$ A^{h} = f(Z^h) .$$
|
||||
$$ \hat{a}^{l} = f(\hat{z}^l) .$$
|
||||
|
||||
<p>
|
||||
This is fed to the output layer:
|
||||
|
||||
$$ Z^{o} = A^{h} W^{o} + B^{o} .$$
|
||||
$$ \hat{z}^{L} = \hat{a}^{L} \hat{W}^{L} + \hat{b}^{L} .$$
|
||||
|
||||
<p>
|
||||
Finally we receive our output values for each image and each category by passing it through the softmax function:
|
||||
|
||||
$$ output = softmax (Z^{o}) = (n_{inputs}, n_{categories}) .$$
|
||||
$$ output = softmax (\hat{z}^{L}) = (n_{inputs}, n_{categories}) .$$
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># setup the feed-forward pass</span>
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># setup the feed-forward pass, subscript h = hidden layer</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">sigmoid</span>(x):
|
||||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">1/</span>(<span style="color: #666666">1</span> <span style="color: #666666">+</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x))
|
||||
@@ -329,7 +330,7 @@ predictions <span style="color: #666666">=</span> predict(X_train)
|
||||
<li><a href="._NeuralNet-bs048.html">49</a></li>
|
||||
<li><a href="._NeuralNet-bs049.html">50</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs041.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -296,7 +297,7 @@ A full derivation is given in the appendix at the end.
|
||||
<li><a href="._NeuralNet-bs049.html">50</a></li>
|
||||
<li><a href="._NeuralNet-bs050.html">51</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs042.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -291,7 +292,7 @@ This has two important benefits:
|
||||
<li><a href="._NeuralNet-bs050.html">51</a></li>
|
||||
<li><a href="._NeuralNet-bs051.html">52</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs043.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -287,7 +288,7 @@ calculate the gradient efficently.
|
||||
<li><a href="._NeuralNet-bs051.html">52</a></li>
|
||||
<li><a href="._NeuralNet-bs052.html">53</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs044.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -237,38 +238,38 @@ MathJax.Hub.Config({
|
||||
To more efficently train our network these equations are implemented using matrix operations.
|
||||
The error in the output layer is calculated simply as
|
||||
|
||||
$$ \Delta_o = \hat{y} - y = (n_{inputs}, n_{categories}) .$$
|
||||
$$ \delta_L = \hat{y} - y = (n_{inputs}, n_{categories}) .$$
|
||||
|
||||
<p>
|
||||
The gradient for the output weights is calculated as
|
||||
|
||||
$$ \nabla W_{o} = A^T \Delta_o = (n_{hidden}, n_{categories}) ,$$
|
||||
$$ \nabla W_{L} = \hat{a}^T \delta_L = (n_{hidden}, n_{categories}) ,$$
|
||||
|
||||
<p>
|
||||
where \( A = (n_{inputs}, n_{hidden}) \). This simply means that we are summing up the gradients for each input.
|
||||
where \( \hat{a} = (n_{inputs}, n_{hidden}) \). This simply means that we are summing up the gradients for each input.
|
||||
Since we are going backwards we have to transpose the activation matrix.
|
||||
|
||||
<p>
|
||||
The gradient with respect to the output bias is then
|
||||
|
||||
$$ \nabla B_{o} = \sum_{i=1}^{n_{inputs}} \Delta_o = (n_{categories}) .$$
|
||||
$$ \nabla \hat{b}_{L} = \sum_{i=1}^{n_{inputs}} \delta_L = (n_{categories}) .$$
|
||||
|
||||
<p>
|
||||
The error in the hidden layer is
|
||||
|
||||
$$ \Delta_h = \Delta_o W_{o}^T \circ f'(Z_{h}) = \Delta_o W_{o}^T \circ A_{h} \circ (1 - A_{h}) = (n_{inputs}, n_{hidden}) ,$$
|
||||
$$ \Delta_h = \delta_L W_{L}^T \circ f'(z_{h}) = \delta_L W_{L}^T \circ a_{h} \circ (1 - a_{h}) = (n_{inputs}, n_{hidden}) ,$$
|
||||
|
||||
<p>
|
||||
where \( f'(A_{h}) \) is the derivative of the activation in the hidden layer. The matrix products mean
|
||||
where \( f'(a_{h}) \) is the derivative of the activation in the hidden layer. The matrix products mean
|
||||
that we are summing up the products for each neuron in the output layer. The symbol \( \circ \) denotes
|
||||
the <em>Hadamard product</em>, meaning element-wise multiplication.
|
||||
|
||||
<p>
|
||||
This again gives us the gradients in the hidden layer:
|
||||
|
||||
$$ \nabla W_{h} = X^T \Delta_h = (n_{features}, n_{hidden}) ,$$
|
||||
$$ \nabla W_{h} = X^T \delta_h = (n_{features}, n_{hidden}) ,$$
|
||||
|
||||
$$ \nabla B_{h} = \sum_{i=1}^{n_{inputs}} \Delta_h = (n_{hidden}) .$$
|
||||
$$ \nabla b_{h} = \sum_{i=1}^{n_{inputs}} \delta_h = (n_{hidden}) .$$
|
||||
|
||||
<p>
|
||||
|
||||
@@ -371,7 +372,7 @@ lmbd <span style="color: #666666">=</span> <span style="color: #666666">0.01</sp
|
||||
<li><a href="._NeuralNet-bs052.html">53</a></li>
|
||||
<li><a href="._NeuralNet-bs053.html">54</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs045.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -274,7 +275,7 @@ Andrew Ng goes through some of these considerations in this <a href="https://you
|
||||
<li><a href="._NeuralNet-bs053.html">54</a></li>
|
||||
<li><a href="._NeuralNet-bs054.html">55</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs046.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -366,6 +367,8 @@ being realizations of this object with different hyperparameters. An implementat
|
||||
<li><a href="._NeuralNet-bs053.html">54</a></li>
|
||||
<li><a href="._NeuralNet-bs054.html">55</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs047.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -287,6 +288,9 @@ test_predict <span style="color: #666666">=</span> dnn<span style="color: #66666
|
||||
<li><a href="._NeuralNet-bs053.html">54</a></li>
|
||||
<li><a href="._NeuralNet-bs054.html">55</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs056.html">57</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs048.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -231,22 +232,18 @@ MathJax.Hub.Config({
|
||||
<a name="part0048"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec47" class="anchor">Adjust hyperparameters (if necessary, network architecture </h2>
|
||||
<h2 id="___sec47" class="anchor">Adjust hyperparameters </h2>
|
||||
|
||||
<p>
|
||||
We now perform a grid search to find the optimal hyperparameters for the network.
|
||||
Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around \( 98 \% \) (\( 2 \% \) error rate).
|
||||
Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around \( 98\% \) (\( 2\% \) error rate).
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>eta_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
|
||||
lmbd_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># store the models for later use</span>
|
||||
<span style="color: #408080; font-style: italic"># store the models for later use</span>
|
||||
DNN_numpy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)), dtype<span style="color: #666666">=</span><span style="color: #008000">object</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># grid search</span>
|
||||
@@ -266,44 +263,6 @@ DNN_numpy <span style="color: #666666">=</span> np<span style="color: #666666">.
|
||||
<span style="color: #008000; font-weight: bold">print</span>()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># optional</span>
|
||||
<span style="color: #408080; font-style: italic"># visual representation of grid search</span>
|
||||
<span style="color: #408080; font-style: italic"># uses seaborn heatmap, I believe you can also do this with matplotlib imshow</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
|
||||
|
||||
sns<span style="color: #666666">.</span>set()
|
||||
|
||||
train_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
|
||||
test_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(eta_vals)):
|
||||
<span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(lmbd_vals)):
|
||||
dnn <span style="color: #666666">=</span> DNN_numpy[i][j]
|
||||
|
||||
train_pred <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_train)
|
||||
test_pred <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_test)
|
||||
|
||||
train_accuracy[i][j] <span style="color: #666666">=</span> accuracy_score(Y_train, train_pred)
|
||||
test_accuracy[i][j] <span style="color: #666666">=</span> accuracy_score(Y_test, test_pred)
|
||||
|
||||
|
||||
fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
|
||||
sns<span style="color: #666666">.</span>heatmap(train_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">"viridis"</span>)
|
||||
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">"Training Accuracy"</span>)
|
||||
ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">"$\eta$"</span>)
|
||||
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">"$\lambda$"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
|
||||
sns<span style="color: #666666">.</span>heatmap(test_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">"viridis"</span>)
|
||||
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">"Test Accuracy"</span>)
|
||||
ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">"$\eta$"</span>)
|
||||
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">"$\lambda$"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
@@ -326,6 +285,8 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._NeuralNet-bs053.html">54</a></li>
|
||||
<li><a href="._NeuralNet-bs054.html">55</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs056.html">57</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs049.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -231,41 +232,13 @@ MathJax.Hub.Config({
|
||||
<a name="part0049"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec48" class="anchor">scikit-learn implementation </h2>
|
||||
|
||||
<p>
|
||||
scikit-learn is a machine learning library for Python. It focuses more on traditional machine learning methods, such as regression, clustering, decision trees, etc. As such, it has only two types of neural networks: Multi Layer Perceptron outputting continuous values, <em>MPLRegressor</em>, and Multi Layer Perceptron outputting labels, <em>MLPClassifier</em>. We will see how simple it is to use these classes.
|
||||
|
||||
<p>
|
||||
scikit-learn implements a few improvements from our neural network, such as early stopping, a varying learning rate, different optimization methods, etc. We would therefore expect a better performance overall.
|
||||
<h2 id="___sec48" class="anchor">Visualization </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.neural_network</span> <span style="color: #008000; font-weight: bold">import</span> MLPClassifier
|
||||
|
||||
<span style="color: #408080; font-style: italic"># store models for later use</span>
|
||||
DNN_scikit <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)), dtype<span style="color: #666666">=</span><span style="color: #008000">object</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i, eta <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(eta_vals):
|
||||
<span style="color: #008000; font-weight: bold">for</span> j, lmbd <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(lmbd_vals):
|
||||
dnn <span style="color: #666666">=</span> MLPClassifier(hidden_layer_sizes<span style="color: #666666">=</span>(n_hidden_neurons), activation<span style="color: #666666">=</span><span style="color: #BA2121">'logistic'</span>,
|
||||
alpha<span style="color: #666666">=</span>lmbd, learning_rate_init<span style="color: #666666">=</span>eta, max_iter<span style="color: #666666">=</span>epochs)
|
||||
dnn<span style="color: #666666">.</span>fit(X_train, Y_train)
|
||||
|
||||
DNN_scikit[i][j] <span style="color: #666666">=</span> dnn
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Learning rate = "</span>, eta)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Lambda = "</span>, lmbd)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Accuracy score on test set: "</span>, dnn<span style="color: #666666">.</span>score(X_test, Y_test))
|
||||
<span style="color: #008000; font-weight: bold">print</span>()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># optional</span>
|
||||
<span style="color: #408080; font-style: italic"># visual representation of grid search</span>
|
||||
<span style="color: #408080; font-style: italic"># uses seaborn heatmap, could probably do this in matplotlib</span>
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># visual representation of grid search</span>
|
||||
<span style="color: #408080; font-style: italic"># uses seaborn heatmap, you can also do this with matplotlib imshow</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
|
||||
|
||||
sns<span style="color: #666666">.</span>set()
|
||||
@@ -275,7 +248,7 @@ test_accuracy <span style="color: #666666">=</span> np<span style="color: #66666
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(eta_vals)):
|
||||
<span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(lmbd_vals)):
|
||||
dnn <span style="color: #666666">=</span> DNN_scikit[i][j]
|
||||
dnn <span style="color: #666666">=</span> DNN_numpy[i][j]
|
||||
|
||||
train_pred <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_train)
|
||||
test_pred <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_test)
|
||||
@@ -320,6 +293,8 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._NeuralNet-bs053.html">54</a></li>
|
||||
<li><a href="._NeuralNet-bs054.html">55</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs056.html">57</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs050.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -231,18 +232,42 @@ MathJax.Hub.Config({
|
||||
<a name="part0050"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec49" class="anchor">Building neural networks in Tensorflow and Keras </h2>
|
||||
<h2 id="___sec49" class="anchor">scikit-learn implementation </h2>
|
||||
|
||||
<p>
|
||||
Now we want to build on the experience gained from our neural network implementation in NumPy and scikit-learn
|
||||
and use it to construct a neural network in Tensorflow. Once we have constructed a neural network in NumPy
|
||||
and Tensorflow, building one in Keras is really quite trivial, though the performance may suffer.
|
||||
<b>scikit-learn</b> focuses more
|
||||
on traditional machine learning methods, such as regression,
|
||||
clustering, decision trees, etc. As such, it has only two types of
|
||||
neural networks: Multi Layer Perceptron outputting continuous values,
|
||||
<em>MPLRegressor</em>, and Multi Layer Perceptron outputting labels,
|
||||
<em>MLPClassifier</em>. We will see how simple it is to use these classes.
|
||||
|
||||
<p>
|
||||
In our previous example we used only one hidden layer, and in this we will use two. From this it should be quite
|
||||
clear how to build one using an arbitrary number of hidden layers, using data structures such as Python lists or
|
||||
NumPy arrays.
|
||||
<b>scikit-learn</b> implements a few improvements from our neural network,
|
||||
such as early stopping, a varying learning rate, different
|
||||
optimization methods, etc. We would therefore expect a better
|
||||
performance overall.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.neural_network</span> <span style="color: #008000; font-weight: bold">import</span> MLPClassifier
|
||||
<span style="color: #408080; font-style: italic"># store models for later use</span>
|
||||
DNN_scikit <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)), dtype<span style="color: #666666">=</span><span style="color: #008000">object</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i, eta <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(eta_vals):
|
||||
<span style="color: #008000; font-weight: bold">for</span> j, lmbd <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(lmbd_vals):
|
||||
dnn <span style="color: #666666">=</span> MLPClassifier(hidden_layer_sizes<span style="color: #666666">=</span>(n_hidden_neurons), activation<span style="color: #666666">=</span><span style="color: #BA2121">'logistic'</span>,
|
||||
alpha<span style="color: #666666">=</span>lmbd, learning_rate_init<span style="color: #666666">=</span>eta, max_iter<span style="color: #666666">=</span>epochs)
|
||||
dnn<span style="color: #666666">.</span>fit(X_train, Y_train)
|
||||
|
||||
DNN_scikit[i][j] <span style="color: #666666">=</span> dnn
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Learning rate = "</span>, eta)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Lambda = "</span>, lmbd)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Accuracy score on test set: "</span>, dnn<span style="color: #666666">.</span>score(X_test, Y_test))
|
||||
<span style="color: #008000; font-weight: bold">print</span>()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -264,6 +289,8 @@ NumPy arrays.
|
||||
<li><a href="._NeuralNet-bs053.html">54</a></li>
|
||||
<li><a href="._NeuralNet-bs054.html">55</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs056.html">57</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs051.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -231,47 +232,44 @@ MathJax.Hub.Config({
|
||||
<a name="part0051"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec50" class="anchor">Tensorflow </h2>
|
||||
|
||||
<p>
|
||||
Tensorflow is an open source library machine learning library
|
||||
developed by the Google Brain team for internal use. It was released
|
||||
under the Apache 2.0 open source license in November 9, 2015.
|
||||
|
||||
<p>
|
||||
Tensorflow is a computational framework that allows you to construct
|
||||
machine learning models at different levels of abstraction, from
|
||||
high-level, object-oriented APIs like Keras, down to the C++ kernels
|
||||
that Tensorflow is built upon. The higher levels of abstraction are
|
||||
simpler to use, but less flexible, and our choice of implementation
|
||||
should reflect the problems we are trying to solve.
|
||||
|
||||
<p>
|
||||
<a href="https://www.tensorflow.org/guide/graphs" target="_self">Tensorflow uses</a> so-called graphs to represent your computation
|
||||
in terms of the dependencies between individual operations, such that you first build a Tensorflow <em>graph</em>
|
||||
to represent your model, and then create a Tensorflow <em>session</em> to run the graph.
|
||||
|
||||
<p>
|
||||
In this guide we will analyze the same data as we did in our NumPy and
|
||||
scikit-learn tutorial, gathered from the MNIST database of images. We
|
||||
will give an introduction to the lower level Python Application
|
||||
Program Interfaces (APIs), and see how we use them to build our graph.
|
||||
Then we will build (effectively) the same graph in Keras, to see just
|
||||
how simple solving a machine learning problem can be.
|
||||
|
||||
<p>
|
||||
To install tensorflow on Unix/Linux systems, use pip as
|
||||
<h2 id="___sec50" class="anchor">Visualization </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>pip3 install tensorflow
|
||||
</pre></div>
|
||||
<p>
|
||||
and/or if you use <b>anaconda</b>, just write (or install from the graphical user interface)
|
||||
<p>
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># optional</span>
|
||||
<span style="color: #408080; font-style: italic"># visual representation of grid search</span>
|
||||
<span style="color: #408080; font-style: italic"># uses seaborn heatmap, could probably do this in matplotlib</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>conda install tensorflow
|
||||
sns<span style="color: #666666">.</span>set()
|
||||
|
||||
train_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
|
||||
test_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(eta_vals)):
|
||||
<span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(lmbd_vals)):
|
||||
dnn <span style="color: #666666">=</span> DNN_scikit[i][j]
|
||||
|
||||
train_pred <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_train)
|
||||
test_pred <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_test)
|
||||
|
||||
train_accuracy[i][j] <span style="color: #666666">=</span> accuracy_score(Y_train, train_pred)
|
||||
test_accuracy[i][j] <span style="color: #666666">=</span> accuracy_score(Y_test, test_pred)
|
||||
|
||||
|
||||
fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
|
||||
sns<span style="color: #666666">.</span>heatmap(train_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">"viridis"</span>)
|
||||
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">"Training Accuracy"</span>)
|
||||
ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">"$\eta$"</span>)
|
||||
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">"$\lambda$"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
|
||||
sns<span style="color: #666666">.</span>heatmap(test_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">"viridis"</span>)
|
||||
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">"Test Accuracy"</span>)
|
||||
ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">"$\eta$"</span>)
|
||||
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">"$\lambda$"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
@@ -293,6 +291,8 @@ and/or if you use <b>anaconda</b>, just write (or install from the graphical use
|
||||
<li><a href="._NeuralNet-bs053.html">54</a></li>
|
||||
<li><a href="._NeuralNet-bs054.html">55</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs056.html">57</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs052.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -231,69 +232,18 @@ MathJax.Hub.Config({
|
||||
<a name="part0052"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec51" class="anchor">Collect and pre-process data </h2>
|
||||
<h2 id="___sec51" class="anchor">Building neural networks in Tensorflow and Keras </h2>
|
||||
|
||||
<p>
|
||||
Now we want to build on the experience gained from our neural network implementation in NumPy and scikit-learn
|
||||
and use it to construct a neural network in Tensorflow. Once we have constructed a neural network in NumPy
|
||||
and Tensorflow, building one in Keras is really quite trivial, though the performance may suffer.
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># import necessary packages</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># ensure the same random numbers appear every time</span>
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># display images in notebook</span>
|
||||
<span style="color: #666666">%</span>matplotlib inline
|
||||
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">'figure.figsize'</span>] <span style="color: #666666">=</span> (<span style="color: #666666">12</span>,<span style="color: #666666">12</span>)
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># download MNIST dataset</span>
|
||||
digits <span style="color: #666666">=</span> datasets<span style="color: #666666">.</span>load_digits()
|
||||
|
||||
<span style="color: #408080; font-style: italic"># define inputs and labels</span>
|
||||
inputs <span style="color: #666666">=</span> digits<span style="color: #666666">.</span>images
|
||||
labels <span style="color: #666666">=</span> digits<span style="color: #666666">.</span>target
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"inputs = (n_inputs, pixel_width, pixel_height) = "</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(inputs<span style="color: #666666">.</span>shape))
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"labels = (n_inputs) = "</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(labels<span style="color: #666666">.</span>shape))
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># flatten the image</span>
|
||||
<span style="color: #408080; font-style: italic"># the value -1 means dimension is inferred from the remaining dimensions: 8x8 = 64</span>
|
||||
n_inputs <span style="color: #666666">=</span> <span style="color: #008000">len</span>(inputs)
|
||||
inputs <span style="color: #666666">=</span> inputs<span style="color: #666666">.</span>reshape(n_inputs, <span style="color: #666666">-1</span>)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"X = (n_inputs, n_features) = "</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(inputs<span style="color: #666666">.</span>shape))
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># choose some random images to display</span>
|
||||
indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>arange(n_inputs)
|
||||
random_indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>choice(indices, size<span style="color: #666666">=5</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i, image <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(digits<span style="color: #666666">.</span>images[random_indices]):
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">1</span>, <span style="color: #666666">5</span>, i<span style="color: #666666">+1</span>)
|
||||
plt<span style="color: #666666">.</span>axis(<span style="color: #BA2121">'off'</span>)
|
||||
plt<span style="color: #666666">.</span>imshow(image, cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>gray_r, interpolation<span style="color: #666666">=</span><span style="color: #BA2121">'nearest'</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"Label: </span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121">"</span> <span style="color: #666666">%</span> digits<span style="color: #666666">.</span>target[random_indices[i]])
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
In our previous example we used only one hidden layer, and in this we will use two. From this it should be quite
|
||||
clear how to build one using an arbitrary number of hidden layers, using data structures such as Python lists or
|
||||
NumPy arrays.
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">keras.utils</span> <span style="color: #008000; font-weight: bold">import</span> to_categorical
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
|
||||
<span style="color: #408080; font-style: italic"># one-hot representation of labels</span>
|
||||
labels <span style="color: #666666">=</span> to_categorical(labels)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># split into train and test data</span>
|
||||
train_size <span style="color: #666666">=</span> <span style="color: #666666">0.8</span>
|
||||
test_size <span style="color: #666666">=</span> <span style="color: #666666">1</span> <span style="color: #666666">-</span> train_size
|
||||
X_train, X_test, Y_train, Y_test <span style="color: #666666">=</span> train_test_split(inputs, labels, train_size<span style="color: #666666">=</span>train_size,
|
||||
test_size<span style="color: #666666">=</span>test_size)
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -313,6 +263,8 @@ X_train, X_test, Y_train, Y_test <span style="color: #666666">=</span> train_tes
|
||||
<li><a href="._NeuralNet-bs053.html">54</a></li>
|
||||
<li><a href="._NeuralNet-bs054.html">55</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs056.html">57</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs053.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -231,147 +232,47 @@ MathJax.Hub.Config({
|
||||
<a name="part0053"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec52" class="anchor">Using TensorFlow backend </h2>
|
||||
<h2 id="___sec52" class="anchor">Tensorflow </h2>
|
||||
|
||||
<ol>
|
||||
<li> Define model and architecture</li>
|
||||
<li> Choose cost function and optimizer</li>
|
||||
</ol>
|
||||
<p>
|
||||
Tensorflow is an open source library machine learning library
|
||||
developed by the Google Brain team for internal use. It was released
|
||||
under the Apache 2.0 open source license in November 9, 2015.
|
||||
|
||||
<p>
|
||||
Tensorflow is a computational framework that allows you to construct
|
||||
machine learning models at different levels of abstraction, from
|
||||
high-level, object-oriented APIs like Keras, down to the C++ kernels
|
||||
that Tensorflow is built upon. The higher levels of abstraction are
|
||||
simpler to use, but less flexible, and our choice of implementation
|
||||
should reflect the problems we are trying to solve.
|
||||
|
||||
<p>
|
||||
<a href="https://www.tensorflow.org/guide/graphs" target="_self">Tensorflow uses</a> so-called graphs to represent your computation
|
||||
in terms of the dependencies between individual operations, such that you first build a Tensorflow <em>graph</em>
|
||||
to represent your model, and then create a Tensorflow <em>session</em> to run the graph.
|
||||
|
||||
<p>
|
||||
In this guide we will analyze the same data as we did in our NumPy and
|
||||
scikit-learn tutorial, gathered from the MNIST database of images. We
|
||||
will give an introduction to the lower level Python Application
|
||||
Program Interfaces (APIs), and see how we use them to build our graph.
|
||||
Then we will build (effectively) the same graph in Keras, to see just
|
||||
how simple solving a machine learning problem can be.
|
||||
|
||||
<p>
|
||||
To install tensorflow on Unix/Linux systems, use pip as
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">tensorflow</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">tf</span>
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>pip3 install tensorflow
|
||||
</pre></div>
|
||||
<p>
|
||||
and/or if you use <b>anaconda</b>, just write (or install from the graphical user interface)
|
||||
<p>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">NeuralNetworkTensorflow</span>:
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(
|
||||
<span style="color: #008000">self</span>,
|
||||
X_train,
|
||||
Y_train,
|
||||
X_test,
|
||||
Y_test,
|
||||
n_neurons_layer1<span style="color: #666666">=100</span>,
|
||||
n_neurons_layer2<span style="color: #666666">=50</span>,
|
||||
n_categories<span style="color: #666666">=2</span>,
|
||||
epochs<span style="color: #666666">=10</span>,
|
||||
batch_size<span style="color: #666666">=100</span>,
|
||||
eta<span style="color: #666666">=0.1</span>,
|
||||
lmbd<span style="color: #666666">=0.0</span>,
|
||||
):
|
||||
|
||||
<span style="color: #408080; font-style: italic"># keep track of number of steps</span>
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>global_step <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>Variable(<span style="color: #666666">0</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>int32, trainable<span style="color: #666666">=</span><span style="color: #008000">False</span>, name<span style="color: #666666">=</span><span style="color: #BA2121">'global_step'</span>)
|
||||
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>X_train <span style="color: #666666">=</span> X_train
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>Y_train <span style="color: #666666">=</span> Y_train
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>X_test <span style="color: #666666">=</span> X_test
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>Y_test <span style="color: #666666">=</span> Y_test
|
||||
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>n_inputs <span style="color: #666666">=</span> X_train<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>]
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>n_features <span style="color: #666666">=</span> X_train<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>]
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>n_neurons_layer1 <span style="color: #666666">=</span> n_neurons_layer1
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>n_neurons_layer2 <span style="color: #666666">=</span> n_neurons_layer2
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>n_categories <span style="color: #666666">=</span> n_categories
|
||||
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>epochs <span style="color: #666666">=</span> epochs
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>batch_size <span style="color: #666666">=</span> batch_size
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>iterations <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>n_inputs <span style="color: #666666">//</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>batch_size
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">=</span> eta
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>lmbd <span style="color: #666666">=</span> lmbd
|
||||
|
||||
<span style="color: #408080; font-style: italic"># build network piece by piece</span>
|
||||
<span style="color: #408080; font-style: italic"># name scopes (with) are used to enforce creation of new variables</span>
|
||||
<span style="color: #408080; font-style: italic"># https://www.tensorflow.org/guide/variables</span>
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>create_placeholders()
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>create_DNN()
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>create_loss()
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>create_optimiser()
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>create_accuracy()
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">create_placeholders</span>(<span style="color: #008000">self</span>):
|
||||
<span style="color: #408080; font-style: italic"># placeholders are fine here, but "Datasets" are the preferred method</span>
|
||||
<span style="color: #408080; font-style: italic"># of streaming data into a model</span>
|
||||
<span style="color: #008000; font-weight: bold">with</span> tf<span style="color: #666666">.</span>name_scope(<span style="color: #BA2121">'data'</span>):
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>X <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>placeholder(tf<span style="color: #666666">.</span>float32, shape<span style="color: #666666">=</span>(<span style="color: #008000">None</span>, <span style="color: #008000">self</span><span style="color: #666666">.</span>n_features), name<span style="color: #666666">=</span><span style="color: #BA2121">'X_data'</span>)
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>Y <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>placeholder(tf<span style="color: #666666">.</span>float32, shape<span style="color: #666666">=</span>(<span style="color: #008000">None</span>, <span style="color: #008000">self</span><span style="color: #666666">.</span>n_categories), name<span style="color: #666666">=</span><span style="color: #BA2121">'Y_data'</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">create_DNN</span>(<span style="color: #008000">self</span>):
|
||||
<span style="color: #008000; font-weight: bold">with</span> tf<span style="color: #666666">.</span>name_scope(<span style="color: #BA2121">'DNN'</span>):
|
||||
<span style="color: #408080; font-style: italic"># the weights are stored to calculate regularization loss later</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Fully connected layer 1</span>
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>W_fc1 <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>weight_variable([<span style="color: #008000">self</span><span style="color: #666666">.</span>n_features, <span style="color: #008000">self</span><span style="color: #666666">.</span>n_neurons_layer1], name<span style="color: #666666">=</span><span style="color: #BA2121">'fc1'</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float32)
|
||||
b_fc1 <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>bias_variable([<span style="color: #008000">self</span><span style="color: #666666">.</span>n_neurons_layer1], name<span style="color: #666666">=</span><span style="color: #BA2121">'fc1'</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float32)
|
||||
a_fc1 <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>nn<span style="color: #666666">.</span>sigmoid(tf<span style="color: #666666">.</span>matmul(<span style="color: #008000">self</span><span style="color: #666666">.</span>X, <span style="color: #008000">self</span><span style="color: #666666">.</span>W_fc1) <span style="color: #666666">+</span> b_fc1)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Fully connected layer 2</span>
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>W_fc2 <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>weight_variable([<span style="color: #008000">self</span><span style="color: #666666">.</span>n_neurons_layer1, <span style="color: #008000">self</span><span style="color: #666666">.</span>n_neurons_layer2], name<span style="color: #666666">=</span><span style="color: #BA2121">'fc2'</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float32)
|
||||
b_fc2 <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>bias_variable([<span style="color: #008000">self</span><span style="color: #666666">.</span>n_neurons_layer2], name<span style="color: #666666">=</span><span style="color: #BA2121">'fc2'</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float32)
|
||||
a_fc2 <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>nn<span style="color: #666666">.</span>sigmoid(tf<span style="color: #666666">.</span>matmul(a_fc1, <span style="color: #008000">self</span><span style="color: #666666">.</span>W_fc2) <span style="color: #666666">+</span> b_fc2)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Output layer</span>
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>W_out <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>weight_variable([<span style="color: #008000">self</span><span style="color: #666666">.</span>n_neurons_layer2, <span style="color: #008000">self</span><span style="color: #666666">.</span>n_categories], name<span style="color: #666666">=</span><span style="color: #BA2121">'out'</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float32)
|
||||
b_out <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>bias_variable([<span style="color: #008000">self</span><span style="color: #666666">.</span>n_categories], name<span style="color: #666666">=</span><span style="color: #BA2121">'out'</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float32)
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>z_out <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>matmul(a_fc2, <span style="color: #008000">self</span><span style="color: #666666">.</span>W_out) <span style="color: #666666">+</span> b_out
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">create_loss</span>(<span style="color: #008000">self</span>):
|
||||
<span style="color: #008000; font-weight: bold">with</span> tf<span style="color: #666666">.</span>name_scope(<span style="color: #BA2121">'loss'</span>):
|
||||
softmax_loss <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>reduce_mean(tf<span style="color: #666666">.</span>nn<span style="color: #666666">.</span>softmax_cross_entropy_with_logits_v2(labels<span style="color: #666666">=</span><span style="color: #008000">self</span><span style="color: #666666">.</span>Y, logits<span style="color: #666666">=</span><span style="color: #008000">self</span><span style="color: #666666">.</span>z_out))
|
||||
|
||||
regularizer_loss_fc1 <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>nn<span style="color: #666666">.</span>l2_loss(<span style="color: #008000">self</span><span style="color: #666666">.</span>W_fc1)
|
||||
regularizer_loss_fc2 <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>nn<span style="color: #666666">.</span>l2_loss(<span style="color: #008000">self</span><span style="color: #666666">.</span>W_fc2)
|
||||
regularizer_loss_out <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>nn<span style="color: #666666">.</span>l2_loss(<span style="color: #008000">self</span><span style="color: #666666">.</span>W_out)
|
||||
regularizer_loss <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>lmbd<span style="color: #666666">*</span>(regularizer_loss_fc1 <span style="color: #666666">+</span> regularizer_loss_fc2 <span style="color: #666666">+</span> regularizer_loss_out)
|
||||
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>loss <span style="color: #666666">=</span> softmax_loss <span style="color: #666666">+</span> regularizer_loss
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">create_accuracy</span>(<span style="color: #008000">self</span>):
|
||||
<span style="color: #008000; font-weight: bold">with</span> tf<span style="color: #666666">.</span>name_scope(<span style="color: #BA2121">'accuracy'</span>):
|
||||
probabilities <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>nn<span style="color: #666666">.</span>softmax(<span style="color: #008000">self</span><span style="color: #666666">.</span>z_out)
|
||||
predictions <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>argmax(probabilities, axis<span style="color: #666666">=1</span>)
|
||||
labels <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>argmax(<span style="color: #008000">self</span><span style="color: #666666">.</span>Y, axis<span style="color: #666666">=1</span>)
|
||||
|
||||
correct_predictions <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>equal(predictions, labels)
|
||||
correct_predictions <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>cast(correct_predictions, tf<span style="color: #666666">.</span>float32)
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>accuracy <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>reduce_mean(correct_predictions)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">create_optimiser</span>(<span style="color: #008000">self</span>):
|
||||
<span style="color: #008000; font-weight: bold">with</span> tf<span style="color: #666666">.</span>name_scope(<span style="color: #BA2121">'optimizer'</span>):
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>optimizer <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>train<span style="color: #666666">.</span>GradientDescentOptimizer(learning_rate<span style="color: #666666">=</span><span style="color: #008000">self</span><span style="color: #666666">.</span>eta)<span style="color: #666666">.</span>minimize(<span style="color: #008000">self</span><span style="color: #666666">.</span>loss, global_step<span style="color: #666666">=</span><span style="color: #008000">self</span><span style="color: #666666">.</span>global_step)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">weight_variable</span>(<span style="color: #008000">self</span>, shape, name<span style="color: #666666">=</span><span style="color: #BA2121">''</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float32):
|
||||
initial <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>truncated_normal(shape, stddev<span style="color: #666666">=0.1</span>)
|
||||
<span style="color: #008000; font-weight: bold">return</span> tf<span style="color: #666666">.</span>Variable(initial, name<span style="color: #666666">=</span>name, dtype<span style="color: #666666">=</span>dtype)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">bias_variable</span>(<span style="color: #008000">self</span>, shape, name<span style="color: #666666">=</span><span style="color: #BA2121">''</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float32):
|
||||
initial <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>constant(<span style="color: #666666">0.1</span>, shape<span style="color: #666666">=</span>shape)
|
||||
<span style="color: #008000; font-weight: bold">return</span> tf<span style="color: #666666">.</span>Variable(initial, name<span style="color: #666666">=</span>name, dtype<span style="color: #666666">=</span>dtype)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">fit</span>(<span style="color: #008000">self</span>):
|
||||
data_indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>arange(<span style="color: #008000">self</span><span style="color: #666666">.</span>n_inputs)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">with</span> tf<span style="color: #666666">.</span>Session() <span style="color: #008000; font-weight: bold">as</span> sess:
|
||||
sess<span style="color: #666666">.</span>run(tf<span style="color: #666666">.</span>global_variables_initializer())
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>epochs):
|
||||
<span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>iterations):
|
||||
chosen_datapoints <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>choice(data_indices, size<span style="color: #666666">=</span><span style="color: #008000">self</span><span style="color: #666666">.</span>batch_size, replace<span style="color: #666666">=</span><span style="color: #008000">False</span>)
|
||||
batch_X, batch_Y <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>X_train[chosen_datapoints], <span style="color: #008000">self</span><span style="color: #666666">.</span>Y_train[chosen_datapoints]
|
||||
|
||||
sess<span style="color: #666666">.</span>run([DNN<span style="color: #666666">.</span>loss, DNN<span style="color: #666666">.</span>optimizer],
|
||||
feed_dict<span style="color: #666666">=</span>{DNN<span style="color: #666666">.</span>X: batch_X,
|
||||
DNN<span style="color: #666666">.</span>Y: batch_Y})
|
||||
accuracy <span style="color: #666666">=</span> sess<span style="color: #666666">.</span>run(DNN<span style="color: #666666">.</span>accuracy,
|
||||
feed_dict<span style="color: #666666">=</span>{DNN<span style="color: #666666">.</span>X: batch_X,
|
||||
DNN<span style="color: #666666">.</span>Y: batch_Y})
|
||||
step <span style="color: #666666">=</span> sess<span style="color: #666666">.</span>run(DNN<span style="color: #666666">.</span>global_step)
|
||||
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>train_loss, <span style="color: #008000">self</span><span style="color: #666666">.</span>train_accuracy <span style="color: #666666">=</span> sess<span style="color: #666666">.</span>run([DNN<span style="color: #666666">.</span>loss, DNN<span style="color: #666666">.</span>accuracy],
|
||||
feed_dict<span style="color: #666666">=</span>{DNN<span style="color: #666666">.</span>X: <span style="color: #008000">self</span><span style="color: #666666">.</span>X_train,
|
||||
DNN<span style="color: #666666">.</span>Y: <span style="color: #008000">self</span><span style="color: #666666">.</span>Y_train})
|
||||
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>test_loss, <span style="color: #008000">self</span><span style="color: #666666">.</span>test_accuracy <span style="color: #666666">=</span> sess<span style="color: #666666">.</span>run([DNN<span style="color: #666666">.</span>loss, DNN<span style="color: #666666">.</span>accuracy],
|
||||
feed_dict<span style="color: #666666">=</span>{DNN<span style="color: #666666">.</span>X: <span style="color: #008000">self</span><span style="color: #666666">.</span>X_test,
|
||||
DNN<span style="color: #666666">.</span>Y: <span style="color: #008000">self</span><span style="color: #666666">.</span>Y_test})
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>conda install tensorflow
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
@@ -391,6 +292,8 @@ MathJax.Hub.Config({
|
||||
<li class="active"><a href="._NeuralNet-bs053.html">54</a></li>
|
||||
<li><a href="._NeuralNet-bs054.html">55</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs056.html">57</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs054.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -231,82 +232,68 @@ MathJax.Hub.Config({
|
||||
<a name="part0054"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec53" class="anchor">Optimizing and using gradient descent </h2>
|
||||
<h2 id="___sec53" class="anchor">Collect and pre-process data </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>epochs <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
batch_size <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
n_neurons_layer1 <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
n_neurons_layer2 <span style="color: #666666">=</span> <span style="color: #666666">50</span>
|
||||
n_categories <span style="color: #666666">=</span> <span style="color: #666666">10</span>
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># import necessary packages</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets
|
||||
|
||||
eta_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
|
||||
lmbd_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>DNN_tf <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)), dtype<span style="color: #666666">=</span><span style="color: #008000">object</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i, eta <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(eta_vals):
|
||||
<span style="color: #008000; font-weight: bold">for</span> j, lmbd <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(lmbd_vals):
|
||||
DNN <span style="color: #666666">=</span> NeuralNetworkTensorflow(X_train, Y_train, X_test, Y_test,
|
||||
n_neurons_layer1, n_neurons_layer2, n_categories,
|
||||
epochs<span style="color: #666666">=</span>epochs, batch_size<span style="color: #666666">=</span>batch_size, eta<span style="color: #666666">=</span>eta, lmbd<span style="color: #666666">=</span>lmbd)
|
||||
DNN<span style="color: #666666">.</span>fit()
|
||||
|
||||
DNN_tf[i][j] <span style="color: #666666">=</span> DNN
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Learning rate = "</span>, eta)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Lambda = "</span>, lmbd)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test accuracy: </span><span style="color: #BB6688; font-weight: bold">%.3f</span><span style="color: #BA2121">"</span> <span style="color: #666666">%</span> DNN<span style="color: #666666">.</span>test_accuracy)
|
||||
<span style="color: #008000; font-weight: bold">print</span>()
|
||||
|
||||
</pre></div>
|
||||
<p>
|
||||
<span style="color: #408080; font-style: italic"># ensure the same random numbers appear every time</span>
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># optional</span>
|
||||
<span style="color: #408080; font-style: italic"># visual representation of grid search</span>
|
||||
<span style="color: #408080; font-style: italic"># uses seaborn heatmap, could probably do this in matplotlib</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
|
||||
<span style="color: #408080; font-style: italic"># display images in notebook</span>
|
||||
<span style="color: #666666">%</span>matplotlib inline
|
||||
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">'figure.figsize'</span>] <span style="color: #666666">=</span> (<span style="color: #666666">12</span>,<span style="color: #666666">12</span>)
|
||||
|
||||
sns<span style="color: #666666">.</span>set()
|
||||
|
||||
train_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
|
||||
test_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
|
||||
<span style="color: #408080; font-style: italic"># download MNIST dataset</span>
|
||||
digits <span style="color: #666666">=</span> datasets<span style="color: #666666">.</span>load_digits()
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(eta_vals)):
|
||||
<span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(lmbd_vals)):
|
||||
DNN <span style="color: #666666">=</span> DNN_tf[i][j]
|
||||
<span style="color: #408080; font-style: italic"># define inputs and labels</span>
|
||||
inputs <span style="color: #666666">=</span> digits<span style="color: #666666">.</span>images
|
||||
labels <span style="color: #666666">=</span> digits<span style="color: #666666">.</span>target
|
||||
|
||||
train_accuracy[i][j] <span style="color: #666666">=</span> DNN<span style="color: #666666">.</span>train_accuracy
|
||||
test_accuracy[i][j] <span style="color: #666666">=</span> DNN<span style="color: #666666">.</span>test_accuracy
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"inputs = (n_inputs, pixel_width, pixel_height) = "</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(inputs<span style="color: #666666">.</span>shape))
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"labels = (n_inputs) = "</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(labels<span style="color: #666666">.</span>shape))
|
||||
|
||||
|
||||
fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
|
||||
sns<span style="color: #666666">.</span>heatmap(train_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">"viridis"</span>)
|
||||
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">"Training Accuracy"</span>)
|
||||
ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">"$\eta$"</span>)
|
||||
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">"$\lambda$"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
|
||||
sns<span style="color: #666666">.</span>heatmap(test_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">"viridis"</span>)
|
||||
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">"Test Accuracy"</span>)
|
||||
ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">"$\eta$"</span>)
|
||||
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">"$\lambda$"</span>)
|
||||
<span style="color: #408080; font-style: italic"># flatten the image</span>
|
||||
<span style="color: #408080; font-style: italic"># the value -1 means dimension is inferred from the remaining dimensions: 8x8 = 64</span>
|
||||
n_inputs <span style="color: #666666">=</span> <span style="color: #008000">len</span>(inputs)
|
||||
inputs <span style="color: #666666">=</span> inputs<span style="color: #666666">.</span>reshape(n_inputs, <span style="color: #666666">-1</span>)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"X = (n_inputs, n_features) = "</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(inputs<span style="color: #666666">.</span>shape))
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># choose some random images to display</span>
|
||||
indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>arange(n_inputs)
|
||||
random_indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>choice(indices, size<span style="color: #666666">=5</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i, image <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(digits<span style="color: #666666">.</span>images[random_indices]):
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">1</span>, <span style="color: #666666">5</span>, i<span style="color: #666666">+1</span>)
|
||||
plt<span style="color: #666666">.</span>axis(<span style="color: #BA2121">'off'</span>)
|
||||
plt<span style="color: #666666">.</span>imshow(image, cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>gray_r, interpolation<span style="color: #666666">=</span><span style="color: #BA2121">'nearest'</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"Label: </span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121">"</span> <span style="color: #666666">%</span> digits<span style="color: #666666">.</span>target[random_indices[i]])
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># optional</span>
|
||||
<span style="color: #408080; font-style: italic"># we can use log files to visualize our graph in Tensorboard</span>
|
||||
writer <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>summary<span style="color: #666666">.</span>FileWriter(<span style="color: #BA2121">'logs/'</span>)
|
||||
writer<span style="color: #666666">.</span>add_graph(tf<span style="color: #666666">.</span>get_default_graph())
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">keras.utils</span> <span style="color: #008000; font-weight: bold">import</span> to_categorical
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
|
||||
<span style="color: #408080; font-style: italic"># one-hot representation of labels</span>
|
||||
labels <span style="color: #666666">=</span> to_categorical(labels)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># split into train and test data</span>
|
||||
train_size <span style="color: #666666">=</span> <span style="color: #666666">0.8</span>
|
||||
test_size <span style="color: #666666">=</span> <span style="color: #666666">1</span> <span style="color: #666666">-</span> train_size
|
||||
X_train, X_test, Y_train, Y_test <span style="color: #666666">=</span> train_test_split(inputs, labels, train_size<span style="color: #666666">=</span>train_size,
|
||||
test_size<span style="color: #666666">=</span>test_size)
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
@@ -325,6 +312,8 @@ writer<span style="color: #666666">.</span>add_graph(tf<span style="color: #6666
|
||||
<li><a href="._NeuralNet-bs053.html">54</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs054.html">55</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs056.html">57</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -231,104 +232,149 @@ MathJax.Hub.Config({
|
||||
<a name="part0055"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec54" class="anchor">Using Keras </h2>
|
||||
<h2 id="___sec54" class="anchor">Using TensorFlow backend </h2>
|
||||
|
||||
<p>
|
||||
Keras is a high level <a href="https://en.wikipedia.org/wiki/Application_programming_interface" target="_self">neural network</a>
|
||||
that supports Tensorflow, CTNK and Theano as backends.
|
||||
If you have Tensorflow installed Keras is available through the <em>tf.keras</em> module.
|
||||
If you have Anaconda installed you may run the following command
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>conda install keras
|
||||
</pre></div>
|
||||
<p>
|
||||
Alternatively, if you have Tensorflow or one of the other supported backends install you may use the pip package manager:
|
||||
<ol>
|
||||
<li> Define model and architecture</li>
|
||||
<li> Choose cost function and optimizer</li>
|
||||
</ol>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>pip3 install keras
|
||||
</pre></div>
|
||||
<p>
|
||||
or look up the <a href="https://keras.io/" target="_self">instructions here</a>.
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">tensorflow</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">tf</span>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">keras.models</span> <span style="color: #008000; font-weight: bold">import</span> Sequential
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">keras.layers</span> <span style="color: #008000; font-weight: bold">import</span> Dense
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">keras.regularizers</span> <span style="color: #008000; font-weight: bold">import</span> l2
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">keras.optimizers</span> <span style="color: #008000; font-weight: bold">import</span> SGD
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">create_neural_network_keras</span>(n_neurons_layer1, n_neurons_layer2, n_categories, eta, lmbd):
|
||||
model <span style="color: #666666">=</span> Sequential()
|
||||
model<span style="color: #666666">.</span>add(Dense(n_neurons_layer1, activation<span style="color: #666666">=</span><span style="color: #BA2121">'sigmoid'</span>, kernel_regularizer<span style="color: #666666">=</span>l2(lmbd)))
|
||||
model<span style="color: #666666">.</span>add(Dense(n_neurons_layer2, activation<span style="color: #666666">=</span><span style="color: #BA2121">'sigmoid'</span>, kernel_regularizer<span style="color: #666666">=</span>l2(lmbd)))
|
||||
model<span style="color: #666666">.</span>add(Dense(n_categories, activation<span style="color: #666666">=</span><span style="color: #BA2121">'softmax'</span>))
|
||||
<span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">NeuralNetworkTensorflow</span>:
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(
|
||||
<span style="color: #008000">self</span>,
|
||||
X_train,
|
||||
Y_train,
|
||||
X_test,
|
||||
Y_test,
|
||||
n_neurons_layer1<span style="color: #666666">=100</span>,
|
||||
n_neurons_layer2<span style="color: #666666">=50</span>,
|
||||
n_categories<span style="color: #666666">=2</span>,
|
||||
epochs<span style="color: #666666">=10</span>,
|
||||
batch_size<span style="color: #666666">=100</span>,
|
||||
eta<span style="color: #666666">=0.1</span>,
|
||||
lmbd<span style="color: #666666">=0.0</span>,
|
||||
):
|
||||
|
||||
<span style="color: #408080; font-style: italic"># keep track of number of steps</span>
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>global_step <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>Variable(<span style="color: #666666">0</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>int32, trainable<span style="color: #666666">=</span><span style="color: #008000">False</span>, name<span style="color: #666666">=</span><span style="color: #BA2121">'global_step'</span>)
|
||||
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>X_train <span style="color: #666666">=</span> X_train
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>Y_train <span style="color: #666666">=</span> Y_train
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>X_test <span style="color: #666666">=</span> X_test
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>Y_test <span style="color: #666666">=</span> Y_test
|
||||
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>n_inputs <span style="color: #666666">=</span> X_train<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>]
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>n_features <span style="color: #666666">=</span> X_train<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>]
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>n_neurons_layer1 <span style="color: #666666">=</span> n_neurons_layer1
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>n_neurons_layer2 <span style="color: #666666">=</span> n_neurons_layer2
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>n_categories <span style="color: #666666">=</span> n_categories
|
||||
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>epochs <span style="color: #666666">=</span> epochs
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>batch_size <span style="color: #666666">=</span> batch_size
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>iterations <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>n_inputs <span style="color: #666666">//</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>batch_size
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">=</span> eta
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>lmbd <span style="color: #666666">=</span> lmbd
|
||||
|
||||
<span style="color: #408080; font-style: italic"># build network piece by piece</span>
|
||||
<span style="color: #408080; font-style: italic"># name scopes (with) are used to enforce creation of new variables</span>
|
||||
<span style="color: #408080; font-style: italic"># https://www.tensorflow.org/guide/variables</span>
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>create_placeholders()
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>create_DNN()
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>create_loss()
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>create_optimiser()
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>create_accuracy()
|
||||
|
||||
sgd <span style="color: #666666">=</span> SGD(lr<span style="color: #666666">=</span>eta)
|
||||
model<span style="color: #666666">.</span>compile(loss<span style="color: #666666">=</span><span style="color: #BA2121">'categorical_crossentropy'</span>, optimizer<span style="color: #666666">=</span>sgd, metrics<span style="color: #666666">=</span>[<span style="color: #BA2121">'accuracy'</span>])
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">create_placeholders</span>(<span style="color: #008000">self</span>):
|
||||
<span style="color: #408080; font-style: italic"># placeholders are fine here, but "Datasets" are the preferred method</span>
|
||||
<span style="color: #408080; font-style: italic"># of streaming data into a model</span>
|
||||
<span style="color: #008000; font-weight: bold">with</span> tf<span style="color: #666666">.</span>name_scope(<span style="color: #BA2121">'data'</span>):
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>X <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>placeholder(tf<span style="color: #666666">.</span>float32, shape<span style="color: #666666">=</span>(<span style="color: #008000">None</span>, <span style="color: #008000">self</span><span style="color: #666666">.</span>n_features), name<span style="color: #666666">=</span><span style="color: #BA2121">'X_data'</span>)
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>Y <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>placeholder(tf<span style="color: #666666">.</span>float32, shape<span style="color: #666666">=</span>(<span style="color: #008000">None</span>, <span style="color: #008000">self</span><span style="color: #666666">.</span>n_categories), name<span style="color: #666666">=</span><span style="color: #BA2121">'Y_data'</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> model
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">create_DNN</span>(<span style="color: #008000">self</span>):
|
||||
<span style="color: #008000; font-weight: bold">with</span> tf<span style="color: #666666">.</span>name_scope(<span style="color: #BA2121">'DNN'</span>):
|
||||
<span style="color: #408080; font-style: italic"># the weights are stored to calculate regularization loss later</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Fully connected layer 1</span>
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>W_fc1 <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>weight_variable([<span style="color: #008000">self</span><span style="color: #666666">.</span>n_features, <span style="color: #008000">self</span><span style="color: #666666">.</span>n_neurons_layer1], name<span style="color: #666666">=</span><span style="color: #BA2121">'fc1'</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float32)
|
||||
b_fc1 <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>bias_variable([<span style="color: #008000">self</span><span style="color: #666666">.</span>n_neurons_layer1], name<span style="color: #666666">=</span><span style="color: #BA2121">'fc1'</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float32)
|
||||
a_fc1 <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>nn<span style="color: #666666">.</span>sigmoid(tf<span style="color: #666666">.</span>matmul(<span style="color: #008000">self</span><span style="color: #666666">.</span>X, <span style="color: #008000">self</span><span style="color: #666666">.</span>W_fc1) <span style="color: #666666">+</span> b_fc1)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Fully connected layer 2</span>
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>W_fc2 <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>weight_variable([<span style="color: #008000">self</span><span style="color: #666666">.</span>n_neurons_layer1, <span style="color: #008000">self</span><span style="color: #666666">.</span>n_neurons_layer2], name<span style="color: #666666">=</span><span style="color: #BA2121">'fc2'</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float32)
|
||||
b_fc2 <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>bias_variable([<span style="color: #008000">self</span><span style="color: #666666">.</span>n_neurons_layer2], name<span style="color: #666666">=</span><span style="color: #BA2121">'fc2'</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float32)
|
||||
a_fc2 <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>nn<span style="color: #666666">.</span>sigmoid(tf<span style="color: #666666">.</span>matmul(a_fc1, <span style="color: #008000">self</span><span style="color: #666666">.</span>W_fc2) <span style="color: #666666">+</span> b_fc2)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Output layer</span>
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>W_out <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>weight_variable([<span style="color: #008000">self</span><span style="color: #666666">.</span>n_neurons_layer2, <span style="color: #008000">self</span><span style="color: #666666">.</span>n_categories], name<span style="color: #666666">=</span><span style="color: #BA2121">'out'</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float32)
|
||||
b_out <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>bias_variable([<span style="color: #008000">self</span><span style="color: #666666">.</span>n_categories], name<span style="color: #666666">=</span><span style="color: #BA2121">'out'</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float32)
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>z_out <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>matmul(a_fc2, <span style="color: #008000">self</span><span style="color: #666666">.</span>W_out) <span style="color: #666666">+</span> b_out
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">create_loss</span>(<span style="color: #008000">self</span>):
|
||||
<span style="color: #008000; font-weight: bold">with</span> tf<span style="color: #666666">.</span>name_scope(<span style="color: #BA2121">'loss'</span>):
|
||||
softmax_loss <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>reduce_mean(tf<span style="color: #666666">.</span>nn<span style="color: #666666">.</span>softmax_cross_entropy_with_logits_v2(labels<span style="color: #666666">=</span><span style="color: #008000">self</span><span style="color: #666666">.</span>Y, logits<span style="color: #666666">=</span><span style="color: #008000">self</span><span style="color: #666666">.</span>z_out))
|
||||
|
||||
regularizer_loss_fc1 <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>nn<span style="color: #666666">.</span>l2_loss(<span style="color: #008000">self</span><span style="color: #666666">.</span>W_fc1)
|
||||
regularizer_loss_fc2 <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>nn<span style="color: #666666">.</span>l2_loss(<span style="color: #008000">self</span><span style="color: #666666">.</span>W_fc2)
|
||||
regularizer_loss_out <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>nn<span style="color: #666666">.</span>l2_loss(<span style="color: #008000">self</span><span style="color: #666666">.</span>W_out)
|
||||
regularizer_loss <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>lmbd<span style="color: #666666">*</span>(regularizer_loss_fc1 <span style="color: #666666">+</span> regularizer_loss_fc2 <span style="color: #666666">+</span> regularizer_loss_out)
|
||||
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>loss <span style="color: #666666">=</span> softmax_loss <span style="color: #666666">+</span> regularizer_loss
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">create_accuracy</span>(<span style="color: #008000">self</span>):
|
||||
<span style="color: #008000; font-weight: bold">with</span> tf<span style="color: #666666">.</span>name_scope(<span style="color: #BA2121">'accuracy'</span>):
|
||||
probabilities <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>nn<span style="color: #666666">.</span>softmax(<span style="color: #008000">self</span><span style="color: #666666">.</span>z_out)
|
||||
predictions <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>argmax(probabilities, axis<span style="color: #666666">=1</span>)
|
||||
labels <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>argmax(<span style="color: #008000">self</span><span style="color: #666666">.</span>Y, axis<span style="color: #666666">=1</span>)
|
||||
|
||||
correct_predictions <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>equal(predictions, labels)
|
||||
correct_predictions <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>cast(correct_predictions, tf<span style="color: #666666">.</span>float32)
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>accuracy <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>reduce_mean(correct_predictions)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">create_optimiser</span>(<span style="color: #008000">self</span>):
|
||||
<span style="color: #008000; font-weight: bold">with</span> tf<span style="color: #666666">.</span>name_scope(<span style="color: #BA2121">'optimizer'</span>):
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>optimizer <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>train<span style="color: #666666">.</span>GradientDescentOptimizer(learning_rate<span style="color: #666666">=</span><span style="color: #008000">self</span><span style="color: #666666">.</span>eta)<span style="color: #666666">.</span>minimize(<span style="color: #008000">self</span><span style="color: #666666">.</span>loss, global_step<span style="color: #666666">=</span><span style="color: #008000">self</span><span style="color: #666666">.</span>global_step)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">weight_variable</span>(<span style="color: #008000">self</span>, shape, name<span style="color: #666666">=</span><span style="color: #BA2121">''</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float32):
|
||||
initial <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>truncated_normal(shape, stddev<span style="color: #666666">=0.1</span>)
|
||||
<span style="color: #008000; font-weight: bold">return</span> tf<span style="color: #666666">.</span>Variable(initial, name<span style="color: #666666">=</span>name, dtype<span style="color: #666666">=</span>dtype)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">bias_variable</span>(<span style="color: #008000">self</span>, shape, name<span style="color: #666666">=</span><span style="color: #BA2121">''</span>, dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float32):
|
||||
initial <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>constant(<span style="color: #666666">0.1</span>, shape<span style="color: #666666">=</span>shape)
|
||||
<span style="color: #008000; font-weight: bold">return</span> tf<span style="color: #666666">.</span>Variable(initial, name<span style="color: #666666">=</span>name, dtype<span style="color: #666666">=</span>dtype)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">fit</span>(<span style="color: #008000">self</span>):
|
||||
data_indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>arange(<span style="color: #008000">self</span><span style="color: #666666">.</span>n_inputs)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">with</span> tf<span style="color: #666666">.</span>Session() <span style="color: #008000; font-weight: bold">as</span> sess:
|
||||
sess<span style="color: #666666">.</span>run(tf<span style="color: #666666">.</span>global_variables_initializer())
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>epochs):
|
||||
<span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>iterations):
|
||||
chosen_datapoints <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>choice(data_indices, size<span style="color: #666666">=</span><span style="color: #008000">self</span><span style="color: #666666">.</span>batch_size, replace<span style="color: #666666">=</span><span style="color: #008000">False</span>)
|
||||
batch_X, batch_Y <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>X_train[chosen_datapoints], <span style="color: #008000">self</span><span style="color: #666666">.</span>Y_train[chosen_datapoints]
|
||||
|
||||
sess<span style="color: #666666">.</span>run([DNN<span style="color: #666666">.</span>loss, DNN<span style="color: #666666">.</span>optimizer],
|
||||
feed_dict<span style="color: #666666">=</span>{DNN<span style="color: #666666">.</span>X: batch_X,
|
||||
DNN<span style="color: #666666">.</span>Y: batch_Y})
|
||||
accuracy <span style="color: #666666">=</span> sess<span style="color: #666666">.</span>run(DNN<span style="color: #666666">.</span>accuracy,
|
||||
feed_dict<span style="color: #666666">=</span>{DNN<span style="color: #666666">.</span>X: batch_X,
|
||||
DNN<span style="color: #666666">.</span>Y: batch_Y})
|
||||
step <span style="color: #666666">=</span> sess<span style="color: #666666">.</span>run(DNN<span style="color: #666666">.</span>global_step)
|
||||
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>train_loss, <span style="color: #008000">self</span><span style="color: #666666">.</span>train_accuracy <span style="color: #666666">=</span> sess<span style="color: #666666">.</span>run([DNN<span style="color: #666666">.</span>loss, DNN<span style="color: #666666">.</span>accuracy],
|
||||
feed_dict<span style="color: #666666">=</span>{DNN<span style="color: #666666">.</span>X: <span style="color: #008000">self</span><span style="color: #666666">.</span>X_train,
|
||||
DNN<span style="color: #666666">.</span>Y: <span style="color: #008000">self</span><span style="color: #666666">.</span>Y_train})
|
||||
|
||||
<span style="color: #008000">self</span><span style="color: #666666">.</span>test_loss, <span style="color: #008000">self</span><span style="color: #666666">.</span>test_accuracy <span style="color: #666666">=</span> sess<span style="color: #666666">.</span>run([DNN<span style="color: #666666">.</span>loss, DNN<span style="color: #666666">.</span>accuracy],
|
||||
feed_dict<span style="color: #666666">=</span>{DNN<span style="color: #666666">.</span>X: <span style="color: #008000">self</span><span style="color: #666666">.</span>X_test,
|
||||
DNN<span style="color: #666666">.</span>Y: <span style="color: #008000">self</span><span style="color: #666666">.</span>Y_test})
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>DNN_keras <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)), dtype<span style="color: #666666">=</span><span style="color: #008000">object</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i, eta <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(eta_vals):
|
||||
<span style="color: #008000; font-weight: bold">for</span> j, lmbd <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(lmbd_vals):
|
||||
DNN <span style="color: #666666">=</span> create_neural_network_keras(n_neurons_layer1, n_neurons_layer2, n_categories,
|
||||
eta<span style="color: #666666">=</span>eta, lmbd<span style="color: #666666">=</span>lmbd)
|
||||
DNN<span style="color: #666666">.</span>fit(X_train, Y_train, epochs<span style="color: #666666">=</span>epochs, batch_size<span style="color: #666666">=</span>batch_size, verbose<span style="color: #666666">=0</span>)
|
||||
scores <span style="color: #666666">=</span> DNN<span style="color: #666666">.</span>evaluate(X_test, Y_test)
|
||||
|
||||
DNN_keras[i][j] <span style="color: #666666">=</span> DNN
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Learning rate = "</span>, eta)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Lambda = "</span>, lmbd)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test accuracy: </span><span style="color: #BB6688; font-weight: bold">%.3f</span><span style="color: #BA2121">"</span> <span style="color: #666666">%</span> scores[<span style="color: #666666">1</span>])
|
||||
<span style="color: #008000; font-weight: bold">print</span>()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># optional</span>
|
||||
<span style="color: #408080; font-style: italic"># visual representation of grid search</span>
|
||||
<span style="color: #408080; font-style: italic"># uses seaborn heatmap, could probably do this in matplotlib</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
|
||||
|
||||
sns<span style="color: #666666">.</span>set()
|
||||
|
||||
train_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
|
||||
test_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(eta_vals)):
|
||||
<span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(lmbd_vals)):
|
||||
DNN <span style="color: #666666">=</span> DNN_keras[i][j]
|
||||
|
||||
train_accuracy[i][j] <span style="color: #666666">=</span> DNN<span style="color: #666666">.</span>evaluate(X_train, Y_train)[<span style="color: #666666">1</span>]
|
||||
test_accuracy[i][j] <span style="color: #666666">=</span> DNN<span style="color: #666666">.</span>evaluate(X_test, Y_test)[<span style="color: #666666">1</span>]
|
||||
|
||||
|
||||
fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
|
||||
sns<span style="color: #666666">.</span>heatmap(train_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">"viridis"</span>)
|
||||
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">"Training Accuracy"</span>)
|
||||
ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">"$\eta$"</span>)
|
||||
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">"$\lambda$"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
|
||||
sns<span style="color: #666666">.</span>heatmap(test_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">"viridis"</span>)
|
||||
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">"Test Accuracy"</span>)
|
||||
ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">"$\eta$"</span>)
|
||||
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">"$\lambda$"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
@@ -344,6 +390,9 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._NeuralNet-bs053.html">54</a></li>
|
||||
<li><a href="._NeuralNet-bs054.html">55</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs056.html">57</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs056.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
|
||||
@@ -110,20 +110,19 @@ Automatically generated HTML file from DocOnce source
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -208,14 +207,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters (if necessary, network architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -250,7 +251,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Oct 2, 2018</h4></center> <!-- date -->
|
||||
<center><h4>Oct 4, 2018</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
|
||||
@@ -274,7 +275,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._NeuralNet-bs008.html">9</a></li>
|
||||
<li><a href="._NeuralNet-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs055.html">56</a></li>
|
||||
<li><a href="._NeuralNet-bs057.html">58</a></li>
|
||||
<li><a href="._NeuralNet-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -148,7 +148,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p> <br>
|
||||
<center><h4>Oct 2, 2018</h4></center> <!-- date -->
|
||||
<center><h4>Oct 4, 2018</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
|
||||
@@ -387,7 +387,7 @@ $$
|
||||
<p> <br>
|
||||
|
||||
This function receives \( x_i \) as inputs.
|
||||
Here the activation \( z=\sum_{i=1}^n w_ix_i \).
|
||||
Here the activation \( z=(\sum_{i=1}^n w_ix_i+b_i) \).
|
||||
In an FFNN of such neurons, the <em>inputs</em> \( x_i \) are the <em>outputs</em> of
|
||||
the neurons in the preceding layer. Furthermore, an MLP is
|
||||
fully-connected, which means that each neuron receives a weighted sum
|
||||
@@ -549,7 +549,7 @@ is the bias \( b_i^l \) and activation \( y_i^l \) of node \( i \) in layer \( l
|
||||
<p>
|
||||
We have that \( \mathrm{W}_l \) is an \( N_{l-1} \times N_l \) matrix, while \( \hat{b}_l \) and \( \hat{y}_l \) are \( N_l \times 1 \) column vectors.
|
||||
With this notation, the sum becomes a matrix-vector multiplication, and we can write
|
||||
the equation for the activations of hidden layer 2 as
|
||||
the equation for the activations of hidden layer 2 (assuming three nodes for simplicity) as
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{equation}
|
||||
@@ -761,7 +761,7 @@ As a convention it is normal to call a network with one layer of input units,
|
||||
units and one layer of output units as a two-layer network. A network with two layers of hidden units is called a three-layer network etc etc.
|
||||
|
||||
<p>
|
||||
For an MLP there is no direct connection between the output nodes/neurons/units and the input nodes/neurons/units.
|
||||
For an MLP network there is no direct connection between the output nodes/neurons/units and the input nodes/neurons/units.
|
||||
Hereafter we will call the various entities of a layer for nodes.
|
||||
There are also no connections within a single layer.
|
||||
|
||||
@@ -877,11 +877,10 @@ $$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
With the activation function \( \hat{z}^l \) we can in turn define the
|
||||
With the activation values \( \hat{z}^l \) we can in turn define the
|
||||
output of layer \( l \) as \( \hat{a}^l = f(\hat{z}^l) \) where \( f \) is our
|
||||
activation function. In the examples here we will use the sigmoid
|
||||
function discussed in our logistic regression lectures and here as
|
||||
well. We will also use the same activation function \( f \) for all layers
|
||||
function discussed in our logistic regression lectures. We will also use the same activation function \( f \) for all layers
|
||||
and their nodes. It means we have
|
||||
|
||||
<p> <br>
|
||||
@@ -979,7 +978,7 @@ $$
|
||||
|
||||
<p>
|
||||
This is an important expression. The second term on the right handside
|
||||
measures how fast the cost is changing as a function of the $j$th
|
||||
measures how fast the cost function is changing as a function of the $j$th
|
||||
output activation. If, for example, the cost function doesn't depend
|
||||
much on a particular output node \( j \), then \( \delta_j^L \) will be small,
|
||||
which is what we would expect. The first term on the right, measures
|
||||
@@ -991,9 +990,9 @@ Notice that everything in the above equations is easily computed. In
|
||||
particular, we compute \( z_j^L \) while computing the behaviour of the
|
||||
network, and it is only a small additional overhead to compute
|
||||
\( f'(z^L_j) \). The exact form of the derivative with respect to the
|
||||
outpuwill, of course, depend on the form of the cost function.
|
||||
output depends on the form of the cost function.
|
||||
However, provided the cost function is known there should be little
|
||||
trouble computing
|
||||
trouble in calculating
|
||||
|
||||
<p> <br>
|
||||
$$
|
||||
@@ -1039,6 +1038,8 @@ That is, the error \( \delta_j^L \) is exactly equal to the rate of change of th
|
||||
|
||||
<p>
|
||||
We have now three equations that are essential for the computations of the derivatives of the cost function at the output layer. These equations are needed to start the algorithm and they are
|
||||
|
||||
<p>
|
||||
<div class="alert alert-block alert-block alert-text-normal">
|
||||
<b>The starting equations.</b>
|
||||
<p> <br>
|
||||
@@ -1082,7 +1083,7 @@ descent. In this case we say the system learns slowly.
|
||||
|
||||
<p>
|
||||
Another interesting feature is that is when the activation function,
|
||||
represented by the sigmoid function here, is rather flat when towards
|
||||
represented by the sigmoid function here, is rather flat when we move towards
|
||||
its end values \( 0 \) and \( 1 \) (see the above Python codes). In these
|
||||
cases, the derivatives of the activation function will also be close
|
||||
to zero, meaning again that the gradients will be small and the
|
||||
@@ -1205,7 +1206,7 @@ $$
|
||||
|
||||
<p>
|
||||
The parameter \( \eta \) is the learning parameter discussed in connection with the gradient descent methods.
|
||||
Here it is convenient to use stochastic radient descent with mini-batches with an outer loop that steps through multiple epochs of training.
|
||||
Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
|
||||
</section>
|
||||
|
||||
|
||||
@@ -1230,11 +1231,11 @@ calculate.
|
||||
|
||||
<p>
|
||||
For an input \( \boldsymbol{a} \) from the hidden layer, the probability that the input \( \boldsymbol{x} \)
|
||||
is in class 0 or 1 is just:
|
||||
|
||||
is in class 0 or 1 is just. We let \( \theta \) represent the unknown weights and biases to be adjusted by our equations). The variable \( x \)
|
||||
represents our activation values \( z \). We have
|
||||
<p> <br>
|
||||
$$
|
||||
P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) = \frac{1}{1 + \exp (- \boldsymbol{a}^T \boldsymbol{w}_{out})} ,
|
||||
P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) = \frac{1}{1 + \exp (- \boldsymbol{x}} ,
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
@@ -1589,7 +1590,7 @@ The denominator is a normalization factor to ensure the outputs (probabilities)
|
||||
The exponent is just the weighted sum of inputs as before:
|
||||
|
||||
<p> <br>
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i = \boldsymbol{a}^T \boldsymbol{w}_j .$$
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i+b_j.$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
@@ -1610,7 +1611,7 @@ Adding a bias value to the weighted sum of inputs allows the neural network to r
|
||||
of values. Without it, any input with the value 0 will be mapped to zero (before being passed through the activation). The bias unit has an output of 1, and a weight to each neuron \( j \), \( b_j \):
|
||||
|
||||
<p> <br>
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i + 1\cdot b_j = \boldsymbol{a}^T \boldsymbol{w}_j + b_j .$$
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i + 1\cdot b_j.$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
@@ -1642,32 +1643,32 @@ output_bias = np.zeros(n_categories) + <span style="color: #B452CD">0.01</span>
|
||||
|
||||
<p>
|
||||
Denote \( F \) the number of features, \( H \) the number of hidden neurons and \( C \) the number of categories.
|
||||
For each input image we calculate a weighted sum of input features (pixel values) to each neuron \( j \) in the hidden layer:
|
||||
For each input image we calculate a weighted sum of input features (pixel values) to each neuron \( j \) in the hidden layer \( l \):
|
||||
|
||||
<p> <br>
|
||||
$$ z_{j}^{h} = \sum_{i=1}^{F} w_{ij}^{h} x_i + b_{j}^{h} = \boldsymbol{x}^T \boldsymbol{w}_{j}^{h} + b_{j}^{h} ,$$
|
||||
$$ z_{j}^{l} = \sum_{i=1}^{F} w_{ij}^{l} x_i + b_{j}^{l},$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
this is then passed through our activation function
|
||||
|
||||
<p> <br>
|
||||
$$ a_{j}^{h} = f(z_{j}^{h}) .$$
|
||||
$$ a_{j}^{l} = f(z_{j}^{l}) .$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
We calculate a weighted sum of inputs (activations in the hidden layer) to each neuron \( j \) in the output layer:
|
||||
|
||||
<p> <br>
|
||||
$$ z_{j}^{o} = \sum_{i=1}^{H} w_{ij}^{o} a_{i}^{h} + b_{j}^{o} = (\boldsymbol{a}^{h})^T \boldsymbol{w}_{j}^{o} + b_{j}^{o} .$$
|
||||
$$ z_{j}^{L} = \sum_{i=1}^{H} w_{ij}^{L} a_{i}^{l} + b_{j}^{L}.$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
Finally we calculate the output of neuron \( j \) in the output layer using the softmax function:
|
||||
|
||||
<p> <br>
|
||||
$$ a_{j}^{o} = \frac{\exp{(z_j^{o})}}
|
||||
{\sum_{c=0}^{C-1} \exp{(z_c^{o})}} .$$
|
||||
$$ a_{j}^{L} = \frac{\exp{(z_j^{L})}}
|
||||
{\sum_{c=0}^{C-1} \exp{(z_c^{L})}} .$$
|
||||
<p> <br>
|
||||
</section>
|
||||
|
||||
@@ -1691,7 +1692,7 @@ for each input image and each hidden neuron.
|
||||
We also add the bias to obtain a matrix of weighted sums to the hidden layer \( Z^{h} \):
|
||||
|
||||
<p> <br>
|
||||
$$ Z^{h} = X W^{h} + B^{h} ,$$
|
||||
$$ \hat{z}^{l} = \hat{X} \hat{W}^{l} + \hat{b}^{l} ,$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
@@ -1699,27 +1700,27 @@ meaning the same bias (1D array with size equal number of hidden neurons) is add
|
||||
This is then passed through the activation:
|
||||
|
||||
<p> <br>
|
||||
$$ A^{h} = f(Z^h) .$$
|
||||
$$ \hat{a}^{l} = f(\hat{z}^l) .$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
This is fed to the output layer:
|
||||
|
||||
<p> <br>
|
||||
$$ Z^{o} = A^{h} W^{o} + B^{o} .$$
|
||||
$$ \hat{z}^{L} = \hat{a}^{L} \hat{W}^{L} + \hat{b}^{L} .$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
Finally we receive our output values for each image and each category by passing it through the softmax function:
|
||||
|
||||
<p> <br>
|
||||
$$ output = softmax (Z^{o}) = (n_{inputs}, n_{categories}) .$$
|
||||
$$ output = softmax (\hat{z}^{L}) = (n_{inputs}, n_{categories}) .$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #228B22"># setup the feed-forward pass</span>
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #228B22"># setup the feed-forward pass, subscript h = hidden layer</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">sigmoid</span>(x):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #B452CD">1</span>/(<span style="color: #B452CD">1</span> + np.exp(-x))
|
||||
@@ -1892,36 +1893,36 @@ To more efficently train our network these equations are implemented using matri
|
||||
The error in the output layer is calculated simply as
|
||||
|
||||
<p> <br>
|
||||
$$ \Delta_o = \hat{y} - y = (n_{inputs}, n_{categories}) .$$
|
||||
$$ \delta_L = \hat{y} - y = (n_{inputs}, n_{categories}) .$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
The gradient for the output weights is calculated as
|
||||
|
||||
<p> <br>
|
||||
$$ \nabla W_{o} = A^T \Delta_o = (n_{hidden}, n_{categories}) ,$$
|
||||
$$ \nabla W_{L} = \hat{a}^T \delta_L = (n_{hidden}, n_{categories}) ,$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
where \( A = (n_{inputs}, n_{hidden}) \). This simply means that we are summing up the gradients for each input.
|
||||
where \( \hat{a} = (n_{inputs}, n_{hidden}) \). This simply means that we are summing up the gradients for each input.
|
||||
Since we are going backwards we have to transpose the activation matrix.
|
||||
|
||||
<p>
|
||||
The gradient with respect to the output bias is then
|
||||
|
||||
<p> <br>
|
||||
$$ \nabla B_{o} = \sum_{i=1}^{n_{inputs}} \Delta_o = (n_{categories}) .$$
|
||||
$$ \nabla \hat{b}_{L} = \sum_{i=1}^{n_{inputs}} \delta_L = (n_{categories}) .$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
The error in the hidden layer is
|
||||
|
||||
<p> <br>
|
||||
$$ \Delta_h = \Delta_o W_{o}^T \circ f'(Z_{h}) = \Delta_o W_{o}^T \circ A_{h} \circ (1 - A_{h}) = (n_{inputs}, n_{hidden}) ,$$
|
||||
$$ \Delta_h = \delta_L W_{L}^T \circ f'(z_{h}) = \delta_L W_{L}^T \circ a_{h} \circ (1 - a_{h}) = (n_{inputs}, n_{hidden}) ,$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
where \( f'(A_{h}) \) is the derivative of the activation in the hidden layer. The matrix products mean
|
||||
where \( f'(a_{h}) \) is the derivative of the activation in the hidden layer. The matrix products mean
|
||||
that we are summing up the products for each neuron in the output layer. The symbol \( \circ \) denotes
|
||||
the <em>Hadamard product</em>, meaning element-wise multiplication.
|
||||
|
||||
@@ -1929,11 +1930,11 @@ the <em>Hadamard product</em>, meaning element-wise multiplication.
|
||||
This again gives us the gradients in the hidden layer:
|
||||
|
||||
<p> <br>
|
||||
$$ \nabla W_{h} = X^T \Delta_h = (n_{features}, n_{hidden}) ,$$
|
||||
$$ \nabla W_{h} = X^T \delta_h = (n_{features}, n_{hidden}) ,$$
|
||||
<p> <br>
|
||||
|
||||
<p> <br>
|
||||
$$ \nabla B_{h} = \sum_{i=1}^{n_{inputs}} \Delta_h = (n_{hidden}) .$$
|
||||
$$ \nabla b_{h} = \sum_{i=1}^{n_{inputs}} \delta_h = (n_{hidden}) .$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
@@ -2187,22 +2188,18 @@ test_predict = dnn.predict(X_test)
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec47">Adjust hyperparameters (if necessary, network architecture </h2>
|
||||
<h2 id="___sec47">Adjust hyperparameters </h2>
|
||||
|
||||
<p>
|
||||
We now perform a grid search to find the optimal hyperparameters for the network.
|
||||
Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around \( 98 \% \) (\( 2 \% \) error rate).
|
||||
Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around \( 98\% \) (\( 2\% \) error rate).
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span>eta_vals = np.logspace(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">7</span>)
|
||||
lmbd_vals = np.logspace(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">7</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #228B22"># store the models for later use</span>
|
||||
<span style="color: #228B22"># store the models for later use</span>
|
||||
DNN_numpy = np.zeros((<span style="color: #658b00">len</span>(eta_vals), <span style="color: #658b00">len</span>(lmbd_vals)), dtype=<span style="color: #658b00">object</span>)
|
||||
|
||||
<span style="color: #228B22"># grid search</span>
|
||||
@@ -2221,12 +2218,17 @@ DNN_numpy = np.zeros((<span style="color: #658b00">len</span>(eta_vals), <span s
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">"Accuracy score on test set: "</span>, accuracy_score(Y_test, test_predict))
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>()
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec48">Visualization </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #228B22"># optional</span>
|
||||
<span style="color: #228B22"># visual representation of grid search</span>
|
||||
<span style="color: #228B22"># uses seaborn heatmap, I believe you can also do this with matplotlib imshow</span>
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #228B22"># visual representation of grid search</span>
|
||||
<span style="color: #228B22"># uses seaborn heatmap, you can also do this with matplotlib imshow</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">seaborn</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">sns</span>
|
||||
|
||||
sns.set()
|
||||
@@ -2263,19 +2265,26 @@ plt.show()
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec48">scikit-learn implementation </h2>
|
||||
<h2 id="___sec49">scikit-learn implementation </h2>
|
||||
|
||||
<p>
|
||||
scikit-learn is a machine learning library for Python. It focuses more on traditional machine learning methods, such as regression, clustering, decision trees, etc. As such, it has only two types of neural networks: Multi Layer Perceptron outputting continuous values, <em>MPLRegressor</em>, and Multi Layer Perceptron outputting labels, <em>MLPClassifier</em>. We will see how simple it is to use these classes.
|
||||
<b>scikit-learn</b> focuses more
|
||||
on traditional machine learning methods, such as regression,
|
||||
clustering, decision trees, etc. As such, it has only two types of
|
||||
neural networks: Multi Layer Perceptron outputting continuous values,
|
||||
<em>MPLRegressor</em>, and Multi Layer Perceptron outputting labels,
|
||||
<em>MLPClassifier</em>. We will see how simple it is to use these classes.
|
||||
|
||||
<p>
|
||||
scikit-learn implements a few improvements from our neural network, such as early stopping, a varying learning rate, different optimization methods, etc. We would therefore expect a better performance overall.
|
||||
<b>scikit-learn</b> implements a few improvements from our neural network,
|
||||
such as early stopping, a varying learning rate, different
|
||||
optimization methods, etc. We would therefore expect a better
|
||||
performance overall.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.neural_network</span> <span style="color: #8B008B; font-weight: bold">import</span> MLPClassifier
|
||||
|
||||
<span style="color: #228B22"># store models for later use</span>
|
||||
DNN_scikit = np.zeros((<span style="color: #658b00">len</span>(eta_vals), <span style="color: #658b00">len</span>(lmbd_vals)), dtype=<span style="color: #658b00">object</span>)
|
||||
|
||||
@@ -2292,6 +2301,11 @@ DNN_scikit = np.zeros((<span style="color: #658b00">len</span>(eta_vals), <span
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">"Accuracy score on test set: "</span>, dnn.score(X_test, Y_test))
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>()
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec50">Visualization </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -2334,7 +2348,7 @@ plt.show()
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec49">Building neural networks in Tensorflow and Keras </h2>
|
||||
<h2 id="___sec51">Building neural networks in Tensorflow and Keras </h2>
|
||||
|
||||
<p>
|
||||
Now we want to build on the experience gained from our neural network implementation in NumPy and scikit-learn
|
||||
@@ -2349,7 +2363,7 @@ NumPy arrays.
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec50">Tensorflow </h2>
|
||||
<h2 id="___sec52">Tensorflow </h2>
|
||||
|
||||
<p>
|
||||
Tensorflow is an open source library machine learning library
|
||||
@@ -2395,7 +2409,7 @@ and/or if you use <b>anaconda</b>, just write (or install from the graphical use
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec51">Collect and pre-process data </h2>
|
||||
<h2 id="___sec53">Collect and pre-process data </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -2462,7 +2476,7 @@ X_train, X_test, Y_train, Y_test = train_test_split(inputs, labels, train_size=t
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec52">Using TensorFlow backend </h2>
|
||||
<h2 id="___sec54">Using TensorFlow backend </h2>
|
||||
|
||||
<ol>
|
||||
<p><li> Define model and architecture</li>
|
||||
@@ -2607,7 +2621,7 @@ X_train, X_test, Y_train, Y_test = train_test_split(inputs, labels, train_size=t
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec53">Optimizing and using gradient descent </h2>
|
||||
<h2 id="___sec55">Optimizing and using gradient descent </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -2617,7 +2631,6 @@ batch_size = <span style="color: #B452CD">100</span>
|
||||
n_neurons_layer1 = <span style="color: #B452CD">100</span>
|
||||
n_neurons_layer2 = <span style="color: #B452CD">50</span>
|
||||
n_categories = <span style="color: #B452CD">10</span>
|
||||
|
||||
eta_vals = np.logspace(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">7</span>)
|
||||
lmbd_vals = np.logspace(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">7</span>)
|
||||
</pre></div>
|
||||
@@ -2639,7 +2652,6 @@ lmbd_vals = np.logspace(-<span style="color: #B452CD">5</span>, <span style="col
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">"Lambda = "</span>, lmbd)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">"Test accuracy: %.3f"</span> % DNN.test_accuracy)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>()
|
||||
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
@@ -2688,7 +2700,7 @@ writer.add_graph(tf.get_default_graph())
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec54">Using Keras </h2>
|
||||
<h2 id="___sec56">Using Keras </h2>
|
||||
|
||||
<p>
|
||||
Keras is a high level <a href="https://en.wikipedia.org/wiki/Application_programming_interface" target="_blank">neural network</a>
|
||||
|
||||
@@ -130,20 +130,19 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -185,7 +184,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Oct 2, 2018</h4></center> <!-- date -->
|
||||
<center><h4>Oct 4, 2018</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
@@ -413,7 +412,7 @@ $$
|
||||
$$
|
||||
|
||||
This function receives \( x_i \) as inputs.
|
||||
Here the activation \( z=\sum_{i=1}^n w_ix_i \).
|
||||
Here the activation \( z=(\sum_{i=1}^n w_ix_i+b_i) \).
|
||||
In an FFNN of such neurons, the <em>inputs</em> \( x_i \) are the <em>outputs</em> of
|
||||
the neurons in the preceding layer. Furthermore, an MLP is
|
||||
fully-connected, which means that each neuron receives a weighted sum
|
||||
@@ -560,7 +559,7 @@ is the bias \( b_i^l \) and activation \( y_i^l \) of node \( i \) in layer \( l
|
||||
<p>
|
||||
We have that \( \mathrm{W}_l \) is an \( N_{l-1} \times N_l \) matrix, while \( \hat{b}_l \) and \( \hat{y}_l \) are \( N_l \times 1 \) column vectors.
|
||||
With this notation, the sum becomes a matrix-vector multiplication, and we can write
|
||||
the equation for the activations of hidden layer 2 as
|
||||
the equation for the activations of hidden layer 2 (assuming three nodes for simplicity) as
|
||||
$$
|
||||
\begin{equation}
|
||||
\hat{y}_2 = f_2(\mathrm{W}_2 \hat{y}_{1} + \hat{b}_{2}) =
|
||||
@@ -753,7 +752,7 @@ As a convention it is normal to call a network with one layer of input units,
|
||||
units and one layer of output units as a two-layer network. A network with two layers of hidden units is called a three-layer network etc etc.
|
||||
|
||||
<p>
|
||||
For an MLP there is no direct connection between the output nodes/neurons/units and the input nodes/neurons/units.
|
||||
For an MLP network there is no direct connection between the output nodes/neurons/units and the input nodes/neurons/units.
|
||||
Hereafter we will call the various entities of a layer for nodes.
|
||||
There are also no connections within a single layer.
|
||||
|
||||
@@ -863,11 +862,10 @@ $$
|
||||
$$
|
||||
|
||||
<p>
|
||||
With the activation function \( \hat{z}^l \) we can in turn define the
|
||||
With the activation values \( \hat{z}^l \) we can in turn define the
|
||||
output of layer \( l \) as \( \hat{a}^l = f(\hat{z}^l) \) where \( f \) is our
|
||||
activation function. In the examples here we will use the sigmoid
|
||||
function discussed in our logistic regression lectures and here as
|
||||
well. We will also use the same activation function \( f \) for all layers
|
||||
function discussed in our logistic regression lectures. We will also use the same activation function \( f \) for all layers
|
||||
and their nodes. It means we have
|
||||
|
||||
$$
|
||||
@@ -945,7 +943,7 @@ $$
|
||||
|
||||
<p>
|
||||
This is an important expression. The second term on the right handside
|
||||
measures how fast the cost is changing as a function of the $j$th
|
||||
measures how fast the cost function is changing as a function of the $j$th
|
||||
output activation. If, for example, the cost function doesn't depend
|
||||
much on a particular output node \( j \), then \( \delta_j^L \) will be small,
|
||||
which is what we would expect. The first term on the right, measures
|
||||
@@ -957,9 +955,9 @@ Notice that everything in the above equations is easily computed. In
|
||||
particular, we compute \( z_j^L \) while computing the behaviour of the
|
||||
network, and it is only a small additional overhead to compute
|
||||
\( f'(z^L_j) \). The exact form of the derivative with respect to the
|
||||
outpuwill, of course, depend on the form of the cost function.
|
||||
output depends on the form of the cost function.
|
||||
However, provided the cost function is known there should be little
|
||||
trouble computing
|
||||
trouble in calculating
|
||||
|
||||
$$
|
||||
\frac{\partial {\cal C}}{\partial (a_j^L)}
|
||||
@@ -995,6 +993,8 @@ That is, the error \( \delta_j^L \) is exactly equal to the rate of change of th
|
||||
|
||||
<p>
|
||||
We have now three equations that are essential for the computations of the derivatives of the cost function at the output layer. These equations are needed to start the algorithm and they are
|
||||
|
||||
<p>
|
||||
<div class="alert alert-block alert-block alert-text-normal">
|
||||
<b>The starting equations.</b>
|
||||
<p>
|
||||
@@ -1035,7 +1035,7 @@ descent. In this case we say the system learns slowly.
|
||||
|
||||
<p>
|
||||
Another interesting feature is that is when the activation function,
|
||||
represented by the sigmoid function here, is rather flat when towards
|
||||
represented by the sigmoid function here, is rather flat when we move towards
|
||||
its end values \( 0 \) and \( 1 \) (see the above Python codes). In these
|
||||
cases, the derivatives of the activation function will also be close
|
||||
to zero, meaning again that the gradients will be small and the
|
||||
@@ -1148,7 +1148,7 @@ $$
|
||||
|
||||
<p>
|
||||
The parameter \( \eta \) is the learning parameter discussed in connection with the gradient descent methods.
|
||||
Here it is convenient to use stochastic radient descent with mini-batches with an outer loop that steps through multiple epochs of training.
|
||||
Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
|
||||
|
||||
<p>
|
||||
<!-- !split -->
|
||||
@@ -1173,10 +1173,10 @@ calculate.
|
||||
|
||||
<p>
|
||||
For an input \( \boldsymbol{a} \) from the hidden layer, the probability that the input \( \boldsymbol{x} \)
|
||||
is in class 0 or 1 is just:
|
||||
|
||||
is in class 0 or 1 is just. We let \( \theta \) represent the unknown weights and biases to be adjusted by our equations). The variable \( x \)
|
||||
represents our activation values \( z \). We have
|
||||
$$
|
||||
P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) = \frac{1}{1 + \exp (- \boldsymbol{a}^T \boldsymbol{w}_{out})} ,
|
||||
P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) = \frac{1}{1 + \exp (- \boldsymbol{x}} ,
|
||||
$$
|
||||
|
||||
and
|
||||
@@ -1494,7 +1494,7 @@ i.e. each neuron \( j \) outputs the probability of being in class \( j \) given
|
||||
The denominator is a normalization factor to ensure the outputs (probabilities) sum up to 1.
|
||||
The exponent is just the weighted sum of inputs as before:
|
||||
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i = \boldsymbol{a}^T \boldsymbol{w}_j .$$
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i+b_j.$$
|
||||
|
||||
<p>
|
||||
Since each neuron in the output layer is connected to the 50 inputs from the hidden layer we have 50x10 = 500
|
||||
@@ -1513,7 +1513,7 @@ or normal distribution. Setting all weights to zero means all neurons give the s
|
||||
Adding a bias value to the weighted sum of inputs allows the neural network to represent a greater range
|
||||
of values. Without it, any input with the value 0 will be mapped to zero (before being passed through the activation). The bias unit has an output of 1, and a weight to each neuron \( j \), \( b_j \):
|
||||
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i + 1\cdot b_j = \boldsymbol{a}^T \boldsymbol{w}_j + b_j .$$
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i + 1\cdot b_j.$$
|
||||
|
||||
<p>
|
||||
The bias weights \( \boldsymbol{b} \) are often initialized to zero, but a small value like \( 0.01 \) ensures all neurons have some output which can be backpropagated in the first training cycle.
|
||||
@@ -1543,25 +1543,25 @@ output_bias = np.zeros(n_categories) + <span style="color: #B452CD">0.01</span>
|
||||
|
||||
<p>
|
||||
Denote \( F \) the number of features, \( H \) the number of hidden neurons and \( C \) the number of categories.
|
||||
For each input image we calculate a weighted sum of input features (pixel values) to each neuron \( j \) in the hidden layer:
|
||||
For each input image we calculate a weighted sum of input features (pixel values) to each neuron \( j \) in the hidden layer \( l \):
|
||||
|
||||
$$ z_{j}^{h} = \sum_{i=1}^{F} w_{ij}^{h} x_i + b_{j}^{h} = \boldsymbol{x}^T \boldsymbol{w}_{j}^{h} + b_{j}^{h} ,$$
|
||||
$$ z_{j}^{l} = \sum_{i=1}^{F} w_{ij}^{l} x_i + b_{j}^{l},$$
|
||||
|
||||
<p>
|
||||
this is then passed through our activation function
|
||||
|
||||
$$ a_{j}^{h} = f(z_{j}^{h}) .$$
|
||||
$$ a_{j}^{l} = f(z_{j}^{l}) .$$
|
||||
|
||||
<p>
|
||||
We calculate a weighted sum of inputs (activations in the hidden layer) to each neuron \( j \) in the output layer:
|
||||
|
||||
$$ z_{j}^{o} = \sum_{i=1}^{H} w_{ij}^{o} a_{i}^{h} + b_{j}^{o} = (\boldsymbol{a}^{h})^T \boldsymbol{w}_{j}^{o} + b_{j}^{o} .$$
|
||||
$$ z_{j}^{L} = \sum_{i=1}^{H} w_{ij}^{L} a_{i}^{l} + b_{j}^{L}.$$
|
||||
|
||||
<p>
|
||||
Finally we calculate the output of neuron \( j \) in the output layer using the softmax function:
|
||||
|
||||
$$ a_{j}^{o} = \frac{\exp{(z_j^{o})}}
|
||||
{\sum_{c=0}^{C-1} \exp{(z_c^{o})}} .$$
|
||||
$$ a_{j}^{L} = \frac{\exp{(z_j^{L})}}
|
||||
{\sum_{c=0}^{C-1} \exp{(z_c^{L})}} .$$
|
||||
|
||||
<p>
|
||||
<!-- !split -->
|
||||
@@ -1581,28 +1581,28 @@ and obtain a matrix that holds the weighted sum of inputs to the hidden layer
|
||||
for each input image and each hidden neuron.
|
||||
We also add the bias to obtain a matrix of weighted sums to the hidden layer \( Z^{h} \):
|
||||
|
||||
$$ Z^{h} = X W^{h} + B^{h} ,$$
|
||||
$$ \hat{z}^{l} = \hat{X} \hat{W}^{l} + \hat{b}^{l} ,$$
|
||||
|
||||
<p>
|
||||
meaning the same bias (1D array with size equal number of hidden neurons) is added to each input image.
|
||||
This is then passed through the activation:
|
||||
|
||||
$$ A^{h} = f(Z^h) .$$
|
||||
$$ \hat{a}^{l} = f(\hat{z}^l) .$$
|
||||
|
||||
<p>
|
||||
This is fed to the output layer:
|
||||
|
||||
$$ Z^{o} = A^{h} W^{o} + B^{o} .$$
|
||||
$$ \hat{z}^{L} = \hat{a}^{L} \hat{W}^{L} + \hat{b}^{L} .$$
|
||||
|
||||
<p>
|
||||
Finally we receive our output values for each image and each category by passing it through the softmax function:
|
||||
|
||||
$$ output = softmax (Z^{o}) = (n_{inputs}, n_{categories}) .$$
|
||||
$$ output = softmax (\hat{z}^{L}) = (n_{inputs}, n_{categories}) .$$
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #228B22"># setup the feed-forward pass</span>
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #228B22"># setup the feed-forward pass, subscript h = hidden layer</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">sigmoid</span>(x):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #B452CD">1</span>/(<span style="color: #B452CD">1</span> + np.exp(-x))
|
||||
@@ -1759,38 +1759,38 @@ calculate the gradient efficently.
|
||||
To more efficently train our network these equations are implemented using matrix operations.
|
||||
The error in the output layer is calculated simply as
|
||||
|
||||
$$ \Delta_o = \hat{y} - y = (n_{inputs}, n_{categories}) .$$
|
||||
$$ \delta_L = \hat{y} - y = (n_{inputs}, n_{categories}) .$$
|
||||
|
||||
<p>
|
||||
The gradient for the output weights is calculated as
|
||||
|
||||
$$ \nabla W_{o} = A^T \Delta_o = (n_{hidden}, n_{categories}) ,$$
|
||||
$$ \nabla W_{L} = \hat{a}^T \delta_L = (n_{hidden}, n_{categories}) ,$$
|
||||
|
||||
<p>
|
||||
where \( A = (n_{inputs}, n_{hidden}) \). This simply means that we are summing up the gradients for each input.
|
||||
where \( \hat{a} = (n_{inputs}, n_{hidden}) \). This simply means that we are summing up the gradients for each input.
|
||||
Since we are going backwards we have to transpose the activation matrix.
|
||||
|
||||
<p>
|
||||
The gradient with respect to the output bias is then
|
||||
|
||||
$$ \nabla B_{o} = \sum_{i=1}^{n_{inputs}} \Delta_o = (n_{categories}) .$$
|
||||
$$ \nabla \hat{b}_{L} = \sum_{i=1}^{n_{inputs}} \delta_L = (n_{categories}) .$$
|
||||
|
||||
<p>
|
||||
The error in the hidden layer is
|
||||
|
||||
$$ \Delta_h = \Delta_o W_{o}^T \circ f'(Z_{h}) = \Delta_o W_{o}^T \circ A_{h} \circ (1 - A_{h}) = (n_{inputs}, n_{hidden}) ,$$
|
||||
$$ \Delta_h = \delta_L W_{L}^T \circ f'(z_{h}) = \delta_L W_{L}^T \circ a_{h} \circ (1 - a_{h}) = (n_{inputs}, n_{hidden}) ,$$
|
||||
|
||||
<p>
|
||||
where \( f'(A_{h}) \) is the derivative of the activation in the hidden layer. The matrix products mean
|
||||
where \( f'(a_{h}) \) is the derivative of the activation in the hidden layer. The matrix products mean
|
||||
that we are summing up the products for each neuron in the output layer. The symbol \( \circ \) denotes
|
||||
the <em>Hadamard product</em>, meaning element-wise multiplication.
|
||||
|
||||
<p>
|
||||
This again gives us the gradients in the hidden layer:
|
||||
|
||||
$$ \nabla W_{h} = X^T \Delta_h = (n_{features}, n_{hidden}) ,$$
|
||||
$$ \nabla W_{h} = X^T \delta_h = (n_{features}, n_{hidden}) ,$$
|
||||
|
||||
$$ \nabla B_{h} = \sum_{i=1}^{n_{inputs}} \Delta_h = (n_{hidden}) .$$
|
||||
$$ \nabla b_{h} = \sum_{i=1}^{n_{inputs}} \delta_h = (n_{hidden}) .$$
|
||||
|
||||
<p>
|
||||
|
||||
@@ -2038,22 +2038,18 @@ test_predict = dnn.predict(X_test)
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec47">Adjust hyperparameters (if necessary, network architecture </h2>
|
||||
<h2 id="___sec47">Adjust hyperparameters </h2>
|
||||
|
||||
<p>
|
||||
We now perform a grid search to find the optimal hyperparameters for the network.
|
||||
Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around \( 98 \% \) (\( 2 \% \) error rate).
|
||||
Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around \( 98\% \) (\( 2\% \) error rate).
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span>eta_vals = np.logspace(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">7</span>)
|
||||
lmbd_vals = np.logspace(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">7</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #228B22"># store the models for later use</span>
|
||||
<span style="color: #228B22"># store the models for later use</span>
|
||||
DNN_numpy = np.zeros((<span style="color: #658b00">len</span>(eta_vals), <span style="color: #658b00">len</span>(lmbd_vals)), dtype=<span style="color: #658b00">object</span>)
|
||||
|
||||
<span style="color: #228B22"># grid search</span>
|
||||
@@ -2072,12 +2068,16 @@ DNN_numpy = np.zeros((<span style="color: #658b00">len</span>(eta_vals), <span s
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">"Accuracy score on test set: "</span>, accuracy_score(Y_test, test_predict))
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>()
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec48">Visualization </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #228B22"># optional</span>
|
||||
<span style="color: #228B22"># visual representation of grid search</span>
|
||||
<span style="color: #228B22"># uses seaborn heatmap, I believe you can also do this with matplotlib imshow</span>
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #228B22"># visual representation of grid search</span>
|
||||
<span style="color: #228B22"># uses seaborn heatmap, you can also do this with matplotlib imshow</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">seaborn</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">sns</span>
|
||||
|
||||
sns.set()
|
||||
@@ -2113,19 +2113,26 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec48">scikit-learn implementation </h2>
|
||||
<h2 id="___sec49">scikit-learn implementation </h2>
|
||||
|
||||
<p>
|
||||
scikit-learn is a machine learning library for Python. It focuses more on traditional machine learning methods, such as regression, clustering, decision trees, etc. As such, it has only two types of neural networks: Multi Layer Perceptron outputting continuous values, <em>MPLRegressor</em>, and Multi Layer Perceptron outputting labels, <em>MLPClassifier</em>. We will see how simple it is to use these classes.
|
||||
<b>scikit-learn</b> focuses more
|
||||
on traditional machine learning methods, such as regression,
|
||||
clustering, decision trees, etc. As such, it has only two types of
|
||||
neural networks: Multi Layer Perceptron outputting continuous values,
|
||||
<em>MPLRegressor</em>, and Multi Layer Perceptron outputting labels,
|
||||
<em>MLPClassifier</em>. We will see how simple it is to use these classes.
|
||||
|
||||
<p>
|
||||
scikit-learn implements a few improvements from our neural network, such as early stopping, a varying learning rate, different optimization methods, etc. We would therefore expect a better performance overall.
|
||||
<b>scikit-learn</b> implements a few improvements from our neural network,
|
||||
such as early stopping, a varying learning rate, different
|
||||
optimization methods, etc. We would therefore expect a better
|
||||
performance overall.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.neural_network</span> <span style="color: #8B008B; font-weight: bold">import</span> MLPClassifier
|
||||
|
||||
<span style="color: #228B22"># store models for later use</span>
|
||||
DNN_scikit = np.zeros((<span style="color: #658b00">len</span>(eta_vals), <span style="color: #658b00">len</span>(lmbd_vals)), dtype=<span style="color: #658b00">object</span>)
|
||||
|
||||
@@ -2143,6 +2150,10 @@ DNN_scikit = np.zeros((<span style="color: #658b00">len</span>(eta_vals), <span
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>()
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec50">Visualization </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #228B22"># optional</span>
|
||||
@@ -2183,7 +2194,7 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec49">Building neural networks in Tensorflow and Keras </h2>
|
||||
<h2 id="___sec51">Building neural networks in Tensorflow and Keras </h2>
|
||||
|
||||
<p>
|
||||
Now we want to build on the experience gained from our neural network implementation in NumPy and scikit-learn
|
||||
@@ -2198,7 +2209,7 @@ NumPy arrays.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec50">Tensorflow </h2>
|
||||
<h2 id="___sec52">Tensorflow </h2>
|
||||
|
||||
<p>
|
||||
Tensorflow is an open source library machine learning library
|
||||
@@ -2243,7 +2254,7 @@ and/or if you use <b>anaconda</b>, just write (or install from the graphical use
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec51">Collect and pre-process data </h2>
|
||||
<h2 id="___sec53">Collect and pre-process data </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -2309,7 +2320,7 @@ X_train, X_test, Y_train, Y_test = train_test_split(inputs, labels, train_size=t
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec52">Using TensorFlow backend </h2>
|
||||
<h2 id="___sec54">Using TensorFlow backend </h2>
|
||||
|
||||
<ol>
|
||||
<li> Define model and architecture</li>
|
||||
@@ -2454,7 +2465,7 @@ X_train, X_test, Y_train, Y_test = train_test_split(inputs, labels, train_size=t
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec53">Optimizing and using gradient descent </h2>
|
||||
<h2 id="___sec55">Optimizing and using gradient descent </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -2464,7 +2475,6 @@ batch_size = <span style="color: #B452CD">100</span>
|
||||
n_neurons_layer1 = <span style="color: #B452CD">100</span>
|
||||
n_neurons_layer2 = <span style="color: #B452CD">50</span>
|
||||
n_categories = <span style="color: #B452CD">10</span>
|
||||
|
||||
eta_vals = np.logspace(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">7</span>)
|
||||
lmbd_vals = np.logspace(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">7</span>)
|
||||
</pre></div>
|
||||
@@ -2486,7 +2496,6 @@ lmbd_vals = np.logspace(-<span style="color: #B452CD">5</span>, <span style="col
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">"Lambda = "</span>, lmbd)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">"Test accuracy: %.3f"</span> % DNN.test_accuracy)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>()
|
||||
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
@@ -2534,7 +2543,7 @@ writer.add_graph(tf.get_default_graph())
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec54">Using Keras </h2>
|
||||
<h2 id="___sec56">Using Keras </h2>
|
||||
|
||||
<p>
|
||||
Keras is a high level <a href="https://en.wikipedia.org/wiki/Application_programming_interface" target="_blank">neural network</a>
|
||||
|
||||
@@ -135,20 +135,19 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
('Improving performance', 2, None, '___sec44'),
|
||||
('Full object-oriented implementation', 2, None, '___sec45'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec46'),
|
||||
('Adjust hyperparameters (if necessary, network architecture',
|
||||
2,
|
||||
None,
|
||||
'___sec47'),
|
||||
('scikit-learn implementation', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec47'),
|
||||
('Visualization', 2, None, '___sec48'),
|
||||
('scikit-learn implementation', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec49'),
|
||||
('Tensorflow', 2, None, '___sec50'),
|
||||
('Collect and pre-process data', 2, None, '___sec51'),
|
||||
('Using TensorFlow backend', 2, None, '___sec52'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec53'),
|
||||
('Using Keras', 2, None, '___sec54')]}
|
||||
'___sec51'),
|
||||
('Tensorflow', 2, None, '___sec52'),
|
||||
('Collect and pre-process data', 2, None, '___sec53'),
|
||||
('Using TensorFlow backend', 2, None, '___sec54'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec55'),
|
||||
('Using Keras', 2, None, '___sec56')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -190,7 +189,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Oct 2, 2018</h4></center> <!-- date -->
|
||||
<center><h4>Oct 4, 2018</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
@@ -418,7 +417,7 @@ $$
|
||||
$$
|
||||
|
||||
This function receives \( x_i \) as inputs.
|
||||
Here the activation \( z=\sum_{i=1}^n w_ix_i \).
|
||||
Here the activation \( z=(\sum_{i=1}^n w_ix_i+b_i) \).
|
||||
In an FFNN of such neurons, the <em>inputs</em> \( x_i \) are the <em>outputs</em> of
|
||||
the neurons in the preceding layer. Furthermore, an MLP is
|
||||
fully-connected, which means that each neuron receives a weighted sum
|
||||
@@ -565,7 +564,7 @@ is the bias \( b_i^l \) and activation \( y_i^l \) of node \( i \) in layer \( l
|
||||
<p>
|
||||
We have that \( \mathrm{W}_l \) is an \( N_{l-1} \times N_l \) matrix, while \( \hat{b}_l \) and \( \hat{y}_l \) are \( N_l \times 1 \) column vectors.
|
||||
With this notation, the sum becomes a matrix-vector multiplication, and we can write
|
||||
the equation for the activations of hidden layer 2 as
|
||||
the equation for the activations of hidden layer 2 (assuming three nodes for simplicity) as
|
||||
$$
|
||||
\begin{equation}
|
||||
\hat{y}_2 = f_2(\mathrm{W}_2 \hat{y}_{1} + \hat{b}_{2}) =
|
||||
@@ -758,7 +757,7 @@ As a convention it is normal to call a network with one layer of input units,
|
||||
units and one layer of output units as a two-layer network. A network with two layers of hidden units is called a three-layer network etc etc.
|
||||
|
||||
<p>
|
||||
For an MLP there is no direct connection between the output nodes/neurons/units and the input nodes/neurons/units.
|
||||
For an MLP network there is no direct connection between the output nodes/neurons/units and the input nodes/neurons/units.
|
||||
Hereafter we will call the various entities of a layer for nodes.
|
||||
There are also no connections within a single layer.
|
||||
|
||||
@@ -868,11 +867,10 @@ $$
|
||||
$$
|
||||
|
||||
<p>
|
||||
With the activation function \( \hat{z}^l \) we can in turn define the
|
||||
With the activation values \( \hat{z}^l \) we can in turn define the
|
||||
output of layer \( l \) as \( \hat{a}^l = f(\hat{z}^l) \) where \( f \) is our
|
||||
activation function. In the examples here we will use the sigmoid
|
||||
function discussed in our logistic regression lectures and here as
|
||||
well. We will also use the same activation function \( f \) for all layers
|
||||
function discussed in our logistic regression lectures. We will also use the same activation function \( f \) for all layers
|
||||
and their nodes. It means we have
|
||||
|
||||
$$
|
||||
@@ -950,7 +948,7 @@ $$
|
||||
|
||||
<p>
|
||||
This is an important expression. The second term on the right handside
|
||||
measures how fast the cost is changing as a function of the $j$th
|
||||
measures how fast the cost function is changing as a function of the $j$th
|
||||
output activation. If, for example, the cost function doesn't depend
|
||||
much on a particular output node \( j \), then \( \delta_j^L \) will be small,
|
||||
which is what we would expect. The first term on the right, measures
|
||||
@@ -962,9 +960,9 @@ Notice that everything in the above equations is easily computed. In
|
||||
particular, we compute \( z_j^L \) while computing the behaviour of the
|
||||
network, and it is only a small additional overhead to compute
|
||||
\( f'(z^L_j) \). The exact form of the derivative with respect to the
|
||||
outpuwill, of course, depend on the form of the cost function.
|
||||
output depends on the form of the cost function.
|
||||
However, provided the cost function is known there should be little
|
||||
trouble computing
|
||||
trouble in calculating
|
||||
|
||||
$$
|
||||
\frac{\partial {\cal C}}{\partial (a_j^L)}
|
||||
@@ -1000,6 +998,8 @@ That is, the error \( \delta_j^L \) is exactly equal to the rate of change of th
|
||||
|
||||
<p>
|
||||
We have now three equations that are essential for the computations of the derivatives of the cost function at the output layer. These equations are needed to start the algorithm and they are
|
||||
|
||||
<p>
|
||||
<div class="alert alert-block alert-block alert-text-normal">
|
||||
<b>The starting equations.</b>
|
||||
<p>
|
||||
@@ -1040,7 +1040,7 @@ descent. In this case we say the system learns slowly.
|
||||
|
||||
<p>
|
||||
Another interesting feature is that is when the activation function,
|
||||
represented by the sigmoid function here, is rather flat when towards
|
||||
represented by the sigmoid function here, is rather flat when we move towards
|
||||
its end values \( 0 \) and \( 1 \) (see the above Python codes). In these
|
||||
cases, the derivatives of the activation function will also be close
|
||||
to zero, meaning again that the gradients will be small and the
|
||||
@@ -1153,7 +1153,7 @@ $$
|
||||
|
||||
<p>
|
||||
The parameter \( \eta \) is the learning parameter discussed in connection with the gradient descent methods.
|
||||
Here it is convenient to use stochastic radient descent with mini-batches with an outer loop that steps through multiple epochs of training.
|
||||
Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
|
||||
|
||||
<p>
|
||||
<!-- !split -->
|
||||
@@ -1178,10 +1178,10 @@ calculate.
|
||||
|
||||
<p>
|
||||
For an input \( \boldsymbol{a} \) from the hidden layer, the probability that the input \( \boldsymbol{x} \)
|
||||
is in class 0 or 1 is just:
|
||||
|
||||
is in class 0 or 1 is just. We let \( \theta \) represent the unknown weights and biases to be adjusted by our equations). The variable \( x \)
|
||||
represents our activation values \( z \). We have
|
||||
$$
|
||||
P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) = \frac{1}{1 + \exp (- \boldsymbol{a}^T \boldsymbol{w}_{out})} ,
|
||||
P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) = \frac{1}{1 + \exp (- \boldsymbol{x}} ,
|
||||
$$
|
||||
|
||||
and
|
||||
@@ -1499,7 +1499,7 @@ i.e. each neuron \( j \) outputs the probability of being in class \( j \) given
|
||||
The denominator is a normalization factor to ensure the outputs (probabilities) sum up to 1.
|
||||
The exponent is just the weighted sum of inputs as before:
|
||||
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i = \boldsymbol{a}^T \boldsymbol{w}_j .$$
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i+b_j.$$
|
||||
|
||||
<p>
|
||||
Since each neuron in the output layer is connected to the 50 inputs from the hidden layer we have 50x10 = 500
|
||||
@@ -1518,7 +1518,7 @@ or normal distribution. Setting all weights to zero means all neurons give the s
|
||||
Adding a bias value to the weighted sum of inputs allows the neural network to represent a greater range
|
||||
of values. Without it, any input with the value 0 will be mapped to zero (before being passed through the activation). The bias unit has an output of 1, and a weight to each neuron \( j \), \( b_j \):
|
||||
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i + 1\cdot b_j = \boldsymbol{a}^T \boldsymbol{w}_j + b_j .$$
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i + 1\cdot b_j.$$
|
||||
|
||||
<p>
|
||||
The bias weights \( \boldsymbol{b} \) are often initialized to zero, but a small value like \( 0.01 \) ensures all neurons have some output which can be backpropagated in the first training cycle.
|
||||
@@ -1548,25 +1548,25 @@ output_bias <span style="color: #666666">=</span> np<span style="color: #666666"
|
||||
|
||||
<p>
|
||||
Denote \( F \) the number of features, \( H \) the number of hidden neurons and \( C \) the number of categories.
|
||||
For each input image we calculate a weighted sum of input features (pixel values) to each neuron \( j \) in the hidden layer:
|
||||
For each input image we calculate a weighted sum of input features (pixel values) to each neuron \( j \) in the hidden layer \( l \):
|
||||
|
||||
$$ z_{j}^{h} = \sum_{i=1}^{F} w_{ij}^{h} x_i + b_{j}^{h} = \boldsymbol{x}^T \boldsymbol{w}_{j}^{h} + b_{j}^{h} ,$$
|
||||
$$ z_{j}^{l} = \sum_{i=1}^{F} w_{ij}^{l} x_i + b_{j}^{l},$$
|
||||
|
||||
<p>
|
||||
this is then passed through our activation function
|
||||
|
||||
$$ a_{j}^{h} = f(z_{j}^{h}) .$$
|
||||
$$ a_{j}^{l} = f(z_{j}^{l}) .$$
|
||||
|
||||
<p>
|
||||
We calculate a weighted sum of inputs (activations in the hidden layer) to each neuron \( j \) in the output layer:
|
||||
|
||||
$$ z_{j}^{o} = \sum_{i=1}^{H} w_{ij}^{o} a_{i}^{h} + b_{j}^{o} = (\boldsymbol{a}^{h})^T \boldsymbol{w}_{j}^{o} + b_{j}^{o} .$$
|
||||
$$ z_{j}^{L} = \sum_{i=1}^{H} w_{ij}^{L} a_{i}^{l} + b_{j}^{L}.$$
|
||||
|
||||
<p>
|
||||
Finally we calculate the output of neuron \( j \) in the output layer using the softmax function:
|
||||
|
||||
$$ a_{j}^{o} = \frac{\exp{(z_j^{o})}}
|
||||
{\sum_{c=0}^{C-1} \exp{(z_c^{o})}} .$$
|
||||
$$ a_{j}^{L} = \frac{\exp{(z_j^{L})}}
|
||||
{\sum_{c=0}^{C-1} \exp{(z_c^{L})}} .$$
|
||||
|
||||
<p>
|
||||
<!-- !split -->
|
||||
@@ -1586,28 +1586,28 @@ and obtain a matrix that holds the weighted sum of inputs to the hidden layer
|
||||
for each input image and each hidden neuron.
|
||||
We also add the bias to obtain a matrix of weighted sums to the hidden layer \( Z^{h} \):
|
||||
|
||||
$$ Z^{h} = X W^{h} + B^{h} ,$$
|
||||
$$ \hat{z}^{l} = \hat{X} \hat{W}^{l} + \hat{b}^{l} ,$$
|
||||
|
||||
<p>
|
||||
meaning the same bias (1D array with size equal number of hidden neurons) is added to each input image.
|
||||
This is then passed through the activation:
|
||||
|
||||
$$ A^{h} = f(Z^h) .$$
|
||||
$$ \hat{a}^{l} = f(\hat{z}^l) .$$
|
||||
|
||||
<p>
|
||||
This is fed to the output layer:
|
||||
|
||||
$$ Z^{o} = A^{h} W^{o} + B^{o} .$$
|
||||
$$ \hat{z}^{L} = \hat{a}^{L} \hat{W}^{L} + \hat{b}^{L} .$$
|
||||
|
||||
<p>
|
||||
Finally we receive our output values for each image and each category by passing it through the softmax function:
|
||||
|
||||
$$ output = softmax (Z^{o}) = (n_{inputs}, n_{categories}) .$$
|
||||
$$ output = softmax (\hat{z}^{L}) = (n_{inputs}, n_{categories}) .$$
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># setup the feed-forward pass</span>
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># setup the feed-forward pass, subscript h = hidden layer</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">sigmoid</span>(x):
|
||||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">1/</span>(<span style="color: #666666">1</span> <span style="color: #666666">+</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x))
|
||||
@@ -1764,38 +1764,38 @@ calculate the gradient efficently.
|
||||
To more efficently train our network these equations are implemented using matrix operations.
|
||||
The error in the output layer is calculated simply as
|
||||
|
||||
$$ \Delta_o = \hat{y} - y = (n_{inputs}, n_{categories}) .$$
|
||||
$$ \delta_L = \hat{y} - y = (n_{inputs}, n_{categories}) .$$
|
||||
|
||||
<p>
|
||||
The gradient for the output weights is calculated as
|
||||
|
||||
$$ \nabla W_{o} = A^T \Delta_o = (n_{hidden}, n_{categories}) ,$$
|
||||
$$ \nabla W_{L} = \hat{a}^T \delta_L = (n_{hidden}, n_{categories}) ,$$
|
||||
|
||||
<p>
|
||||
where \( A = (n_{inputs}, n_{hidden}) \). This simply means that we are summing up the gradients for each input.
|
||||
where \( \hat{a} = (n_{inputs}, n_{hidden}) \). This simply means that we are summing up the gradients for each input.
|
||||
Since we are going backwards we have to transpose the activation matrix.
|
||||
|
||||
<p>
|
||||
The gradient with respect to the output bias is then
|
||||
|
||||
$$ \nabla B_{o} = \sum_{i=1}^{n_{inputs}} \Delta_o = (n_{categories}) .$$
|
||||
$$ \nabla \hat{b}_{L} = \sum_{i=1}^{n_{inputs}} \delta_L = (n_{categories}) .$$
|
||||
|
||||
<p>
|
||||
The error in the hidden layer is
|
||||
|
||||
$$ \Delta_h = \Delta_o W_{o}^T \circ f'(Z_{h}) = \Delta_o W_{o}^T \circ A_{h} \circ (1 - A_{h}) = (n_{inputs}, n_{hidden}) ,$$
|
||||
$$ \Delta_h = \delta_L W_{L}^T \circ f'(z_{h}) = \delta_L W_{L}^T \circ a_{h} \circ (1 - a_{h}) = (n_{inputs}, n_{hidden}) ,$$
|
||||
|
||||
<p>
|
||||
where \( f'(A_{h}) \) is the derivative of the activation in the hidden layer. The matrix products mean
|
||||
where \( f'(a_{h}) \) is the derivative of the activation in the hidden layer. The matrix products mean
|
||||
that we are summing up the products for each neuron in the output layer. The symbol \( \circ \) denotes
|
||||
the <em>Hadamard product</em>, meaning element-wise multiplication.
|
||||
|
||||
<p>
|
||||
This again gives us the gradients in the hidden layer:
|
||||
|
||||
$$ \nabla W_{h} = X^T \Delta_h = (n_{features}, n_{hidden}) ,$$
|
||||
$$ \nabla W_{h} = X^T \delta_h = (n_{features}, n_{hidden}) ,$$
|
||||
|
||||
$$ \nabla B_{h} = \sum_{i=1}^{n_{inputs}} \Delta_h = (n_{hidden}) .$$
|
||||
$$ \nabla b_{h} = \sum_{i=1}^{n_{inputs}} \delta_h = (n_{hidden}) .$$
|
||||
|
||||
<p>
|
||||
|
||||
@@ -2043,22 +2043,18 @@ test_predict <span style="color: #666666">=</span> dnn<span style="color: #66666
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec47">Adjust hyperparameters (if necessary, network architecture </h2>
|
||||
<h2 id="___sec47">Adjust hyperparameters </h2>
|
||||
|
||||
<p>
|
||||
We now perform a grid search to find the optimal hyperparameters for the network.
|
||||
Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around \( 98 \% \) (\( 2 \% \) error rate).
|
||||
Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around \( 98\% \) (\( 2\% \) error rate).
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>eta_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
|
||||
lmbd_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># store the models for later use</span>
|
||||
<span style="color: #408080; font-style: italic"># store the models for later use</span>
|
||||
DNN_numpy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)), dtype<span style="color: #666666">=</span><span style="color: #008000">object</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># grid search</span>
|
||||
@@ -2077,12 +2073,16 @@ DNN_numpy <span style="color: #666666">=</span> np<span style="color: #666666">.
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Accuracy score on test set: "</span>, accuracy_score(Y_test, test_predict))
|
||||
<span style="color: #008000; font-weight: bold">print</span>()
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec48">Visualization </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># optional</span>
|
||||
<span style="color: #408080; font-style: italic"># visual representation of grid search</span>
|
||||
<span style="color: #408080; font-style: italic"># uses seaborn heatmap, I believe you can also do this with matplotlib imshow</span>
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># visual representation of grid search</span>
|
||||
<span style="color: #408080; font-style: italic"># uses seaborn heatmap, you can also do this with matplotlib imshow</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
|
||||
|
||||
sns<span style="color: #666666">.</span>set()
|
||||
@@ -2118,19 +2118,26 @@ plt<span style="color: #666666">.</span>show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec48">scikit-learn implementation </h2>
|
||||
<h2 id="___sec49">scikit-learn implementation </h2>
|
||||
|
||||
<p>
|
||||
scikit-learn is a machine learning library for Python. It focuses more on traditional machine learning methods, such as regression, clustering, decision trees, etc. As such, it has only two types of neural networks: Multi Layer Perceptron outputting continuous values, <em>MPLRegressor</em>, and Multi Layer Perceptron outputting labels, <em>MLPClassifier</em>. We will see how simple it is to use these classes.
|
||||
<b>scikit-learn</b> focuses more
|
||||
on traditional machine learning methods, such as regression,
|
||||
clustering, decision trees, etc. As such, it has only two types of
|
||||
neural networks: Multi Layer Perceptron outputting continuous values,
|
||||
<em>MPLRegressor</em>, and Multi Layer Perceptron outputting labels,
|
||||
<em>MLPClassifier</em>. We will see how simple it is to use these classes.
|
||||
|
||||
<p>
|
||||
scikit-learn implements a few improvements from our neural network, such as early stopping, a varying learning rate, different optimization methods, etc. We would therefore expect a better performance overall.
|
||||
<b>scikit-learn</b> implements a few improvements from our neural network,
|
||||
such as early stopping, a varying learning rate, different
|
||||
optimization methods, etc. We would therefore expect a better
|
||||
performance overall.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.neural_network</span> <span style="color: #008000; font-weight: bold">import</span> MLPClassifier
|
||||
|
||||
<span style="color: #408080; font-style: italic"># store models for later use</span>
|
||||
DNN_scikit <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)), dtype<span style="color: #666666">=</span><span style="color: #008000">object</span>)
|
||||
|
||||
@@ -2148,6 +2155,10 @@ DNN_scikit <span style="color: #666666">=</span> np<span style="color: #666666">
|
||||
<span style="color: #008000; font-weight: bold">print</span>()
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec50">Visualization </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># optional</span>
|
||||
@@ -2188,7 +2199,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec49">Building neural networks in Tensorflow and Keras </h2>
|
||||
<h2 id="___sec51">Building neural networks in Tensorflow and Keras </h2>
|
||||
|
||||
<p>
|
||||
Now we want to build on the experience gained from our neural network implementation in NumPy and scikit-learn
|
||||
@@ -2203,7 +2214,7 @@ NumPy arrays.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec50">Tensorflow </h2>
|
||||
<h2 id="___sec52">Tensorflow </h2>
|
||||
|
||||
<p>
|
||||
Tensorflow is an open source library machine learning library
|
||||
@@ -2248,7 +2259,7 @@ and/or if you use <b>anaconda</b>, just write (or install from the graphical use
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec51">Collect and pre-process data </h2>
|
||||
<h2 id="___sec53">Collect and pre-process data </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -2314,7 +2325,7 @@ X_train, X_test, Y_train, Y_test <span style="color: #666666">=</span> train_tes
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec52">Using TensorFlow backend </h2>
|
||||
<h2 id="___sec54">Using TensorFlow backend </h2>
|
||||
|
||||
<ol>
|
||||
<li> Define model and architecture</li>
|
||||
@@ -2459,7 +2470,7 @@ X_train, X_test, Y_train, Y_test <span style="color: #666666">=</span> train_tes
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec53">Optimizing and using gradient descent </h2>
|
||||
<h2 id="___sec55">Optimizing and using gradient descent </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -2469,7 +2480,6 @@ batch_size <span style="color: #666666">=</span> <span style="color: #666666">10
|
||||
n_neurons_layer1 <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
n_neurons_layer2 <span style="color: #666666">=</span> <span style="color: #666666">50</span>
|
||||
n_categories <span style="color: #666666">=</span> <span style="color: #666666">10</span>
|
||||
|
||||
eta_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
|
||||
lmbd_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
|
||||
</pre></div>
|
||||
@@ -2491,7 +2501,6 @@ lmbd_vals <span style="color: #666666">=</span> np<span style="color: #666666">.
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Lambda = "</span>, lmbd)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test accuracy: </span><span style="color: #BB6688; font-weight: bold">%.3f</span><span style="color: #BA2121">"</span> <span style="color: #666666">%</span> DNN<span style="color: #666666">.</span>test_accuracy)
|
||||
<span style="color: #008000; font-weight: bold">print</span>()
|
||||
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
@@ -2539,7 +2548,7 @@ writer<span style="color: #666666">.</span>add_graph(tf<span style="color: #6666
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec54">Using Keras </h2>
|
||||
<h2 id="___sec56">Using Keras </h2>
|
||||
|
||||
<p>
|
||||
Keras is a high level <a href="https://en.wikipedia.org/wiki/Application_programming_interface" target="_blank">neural network</a>
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"<!-- Author: --> \n",
|
||||
"**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n",
|
||||
"\n",
|
||||
"Date: **Oct 2, 2018**\n",
|
||||
"Date: **Oct 4, 2018**\n",
|
||||
"\n",
|
||||
"Copyright 1999-2018, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license\n",
|
||||
"\n",
|
||||
@@ -222,7 +222,7 @@
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This function receives $x_i$ as inputs.\n",
|
||||
"Here the activation $z=\\sum_{i=1}^n w_ix_i$. \n",
|
||||
"Here the activation $z=(\\sum_{i=1}^n w_ix_i+b_i)$. \n",
|
||||
"In an FFNN of such neurons, the *inputs* $x_i$ are the *outputs* of\n",
|
||||
"the neurons in the preceding layer. Furthermore, an MLP is\n",
|
||||
"fully-connected, which means that each neuron receives a weighted sum\n",
|
||||
@@ -460,7 +460,7 @@
|
||||
"\n",
|
||||
"We have that $\\mathrm{W}_l$ is an $N_{l-1} \\times N_l$ matrix, while $\\hat{b}_l$ and $\\hat{y}_l$ are $N_l \\times 1$ column vectors. \n",
|
||||
"With this notation, the sum becomes a matrix-vector multiplication, and we can write\n",
|
||||
"the equation for the activations of hidden layer 2 as"
|
||||
"the equation for the activations of hidden layer 2 (assuming three nodes for simplicity) as"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -693,7 +693,7 @@
|
||||
"As a convention it is normal to call a network with one layer of input units, one layer of hidden\n",
|
||||
"units and one layer of output units as a two-layer network. A network with two layers of hidden units is called a three-layer network etc etc.\n",
|
||||
"\n",
|
||||
"For an MLP there is no direct connection between the output nodes/neurons/units and the input nodes/neurons/units.\n",
|
||||
"For an MLP network there is no direct connection between the output nodes/neurons/units and the input nodes/neurons/units.\n",
|
||||
"Hereafter we will call the various entities of a layer for nodes.\n",
|
||||
"There are also no connections within a single layer.\n",
|
||||
"\n",
|
||||
@@ -815,11 +815,10 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"With the activation function $\\hat{z}^l$ we can in turn define the\n",
|
||||
"With the activation values $\\hat{z}^l$ we can in turn define the\n",
|
||||
"output of layer $l$ as $\\hat{a}^l = f(\\hat{z}^l)$ where $f$ is our\n",
|
||||
"activation function. In the examples here we will use the sigmoid\n",
|
||||
"function discussed in our logistic regression lectures and here as\n",
|
||||
"well. We will also use the same activation function $f$ for all layers\n",
|
||||
"function discussed in our logistic regression lectures. We will also use the same activation function $f$ for all layers\n",
|
||||
"and their nodes. It means we have"
|
||||
]
|
||||
},
|
||||
@@ -989,7 +988,7 @@
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This is an important expression. The second term on the right handside\n",
|
||||
"measures how fast the cost is changing as a function of the $j$th\n",
|
||||
"measures how fast the cost function is changing as a function of the $j$th\n",
|
||||
"output activation. If, for example, the cost function doesn't depend\n",
|
||||
"much on a particular output node $j$, then $\\delta_j^L$ will be small,\n",
|
||||
"which is what we would expect. The first term on the right, measures\n",
|
||||
@@ -1000,9 +999,9 @@
|
||||
"particular, we compute $z_j^L$ while computing the behaviour of the\n",
|
||||
"network, and it is only a small additional overhead to compute\n",
|
||||
"$f'(z^L_j)$. The exact form of the derivative with respect to the\n",
|
||||
"outpuwill, of course, depend on the form of the cost function.\n",
|
||||
"output depends on the form of the cost function.\n",
|
||||
"However, provided the cost function is known there should be little\n",
|
||||
"trouble computing"
|
||||
"trouble in calculating"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1072,6 +1071,7 @@
|
||||
"## Bringing it together\n",
|
||||
"\n",
|
||||
"We have now three equations that are essential for the computations of the derivatives of the cost function at the output layer. These equations are needed to start the algorithm and they are\n",
|
||||
"\n",
|
||||
"**The starting equations.**"
|
||||
]
|
||||
},
|
||||
@@ -1146,7 +1146,7 @@
|
||||
"descent. In this case we say the system learns slowly.\n",
|
||||
"\n",
|
||||
"Another interesting feature is that is when the activation function,\n",
|
||||
"represented by the sigmoid function here, is rather flat when towards\n",
|
||||
"represented by the sigmoid function here, is rather flat when we move towards\n",
|
||||
"its end values $0$ and $1$ (see the above Python codes). In these\n",
|
||||
"cases, the derivatives of the activation function will also be close\n",
|
||||
"to zero, meaning again that the gradients will be small and the\n",
|
||||
@@ -1306,7 +1306,7 @@
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The parameter $\\eta$ is the learning parameter discussed in connection with the gradient descent methods.\n",
|
||||
"Here it is convenient to use stochastic radient descent with mini-batches with an outer loop that steps through multiple epochs of training.\n",
|
||||
"Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"<!-- !split -->\n",
|
||||
@@ -1327,7 +1327,8 @@
|
||||
"calculate.\n",
|
||||
"\n",
|
||||
"For an input $\\boldsymbol{a}$ from the hidden layer, the probability that the input $\\boldsymbol{x}$\n",
|
||||
"is in class 0 or 1 is just:"
|
||||
"is in class 0 or 1 is just. We let $\\theta$ represent the unknown weights and biases to be adjusted by our equations). The variable $x$\n",
|
||||
"represents our activation values $z$. We have"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1335,7 +1336,7 @@
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"P(y = 0 \\mid \\boldsymbol{x}, \\boldsymbol{\\theta}) = \\frac{1}{1 + \\exp (- \\boldsymbol{a}^T \\boldsymbol{w}_{out})} ,\n",
|
||||
"P(y = 0 \\mid \\boldsymbol{x}, \\boldsymbol{\\theta}) = \\frac{1}{1 + \\exp (- \\boldsymbol{x}} ,\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
@@ -1690,7 +1691,7 @@
|
||||
"The denominator is a normalization factor to ensure the outputs (probabilities) sum up to 1. \n",
|
||||
"The exponent is just the weighted sum of inputs as before: \n",
|
||||
"\n",
|
||||
"$$ z_j = \\sum_{i=1}^n w_ {ij} a_i = \\boldsymbol{a}^T \\boldsymbol{w}_j .$$ \n",
|
||||
"$$ z_j = \\sum_{i=1}^n w_ {ij} a_i+b_j.$$ \n",
|
||||
"\n",
|
||||
"Since each neuron in the output layer is connected to the 50 inputs from the hidden layer we have 50x10 = 500\n",
|
||||
"weights to the output layer.\n",
|
||||
@@ -1704,7 +1705,7 @@
|
||||
"Adding a bias value to the weighted sum of inputs allows the neural network to represent a greater range\n",
|
||||
"of values. Without it, any input with the value 0 will be mapped to zero (before being passed through the activation). The bias unit has an output of 1, and a weight to each neuron $j$, $b_j$: \n",
|
||||
"\n",
|
||||
"$$ z_j = \\sum_{i=1}^n w_ {ij} a_i + 1\\cdot b_j = \\boldsymbol{a}^T \\boldsymbol{w}_j + b_j .$$ \n",
|
||||
"$$ z_j = \\sum_{i=1}^n w_ {ij} a_i + 1\\cdot b_j.$$ \n",
|
||||
"\n",
|
||||
"The bias weights $\\boldsymbol{b}$ are often initialized to zero, but a small value like $0.01$ ensures all neurons have some output which can be backpropagated in the first training cycle."
|
||||
]
|
||||
@@ -1741,22 +1742,22 @@
|
||||
"## Feed-forward pass\n",
|
||||
"\n",
|
||||
"Denote $F$ the number of features, $H$ the number of hidden neurons and $C$ the number of categories. \n",
|
||||
"For each input image we calculate a weighted sum of input features (pixel values) to each neuron $j$ in the hidden layer: \n",
|
||||
"For each input image we calculate a weighted sum of input features (pixel values) to each neuron $j$ in the hidden layer $l$: \n",
|
||||
"\n",
|
||||
"$$ z_{j}^{h} = \\sum_{i=1}^{F} w_{ij}^{h} x_i + b_{j}^{h} = \\boldsymbol{x}^T \\boldsymbol{w}_{j}^{h} + b_{j}^{h} ,$$\n",
|
||||
"$$ z_{j}^{l} = \\sum_{i=1}^{F} w_{ij}^{l} x_i + b_{j}^{l},$$\n",
|
||||
"\n",
|
||||
"this is then passed through our activation function \n",
|
||||
"\n",
|
||||
"$$ a_{j}^{h} = f(z_{j}^{h}) .$$ \n",
|
||||
"$$ a_{j}^{l} = f(z_{j}^{l}) .$$ \n",
|
||||
"\n",
|
||||
"We calculate a weighted sum of inputs (activations in the hidden layer) to each neuron $j$ in the output layer: \n",
|
||||
"\n",
|
||||
"$$ z_{j}^{o} = \\sum_{i=1}^{H} w_{ij}^{o} a_{i}^{h} + b_{j}^{o} = (\\boldsymbol{a}^{h})^T \\boldsymbol{w}_{j}^{o} + b_{j}^{o} .$$ \n",
|
||||
"$$ z_{j}^{L} = \\sum_{i=1}^{H} w_{ij}^{L} a_{i}^{l} + b_{j}^{L}.$$ \n",
|
||||
"\n",
|
||||
"Finally we calculate the output of neuron $j$ in the output layer using the softmax function: \n",
|
||||
"\n",
|
||||
"$$ a_{j}^{o} = \\frac{\\exp{(z_j^{o})}}\n",
|
||||
"{\\sum_{c=0}^{C-1} \\exp{(z_c^{o})}} .$$ \n",
|
||||
"$$ a_{j}^{L} = \\frac{\\exp{(z_j^{L})}}\n",
|
||||
"{\\sum_{c=0}^{C-1} \\exp{(z_c^{L})}} .$$ \n",
|
||||
"\n",
|
||||
"<!-- !split -->\n",
|
||||
"## Matrix multiplication\n",
|
||||
@@ -1772,20 +1773,20 @@
|
||||
"for each input image and each hidden neuron. \n",
|
||||
"We also add the bias to obtain a matrix of weighted sums to the hidden layer $Z^{h}$: \n",
|
||||
"\n",
|
||||
"$$ Z^{h} = X W^{h} + B^{h} ,$$\n",
|
||||
"$$ \\hat{z}^{l} = \\hat{X} \\hat{W}^{l} + \\hat{b}^{l} ,$$\n",
|
||||
"\n",
|
||||
"meaning the same bias (1D array with size equal number of hidden neurons) is added to each input image. \n",
|
||||
"This is then passed through the activation: \n",
|
||||
"\n",
|
||||
"$$ A^{h} = f(Z^h) .$$ \n",
|
||||
"$$ \\hat{a}^{l} = f(\\hat{z}^l) .$$ \n",
|
||||
"\n",
|
||||
"This is fed to the output layer: \n",
|
||||
"\n",
|
||||
"$$ Z^{o} = A^{h} W^{o} + B^{o} .$$\n",
|
||||
"$$ \\hat{z}^{L} = \\hat{a}^{L} \\hat{W}^{L} + \\hat{b}^{L} .$$\n",
|
||||
"\n",
|
||||
"Finally we receive our output values for each image and each category by passing it through the softmax function: \n",
|
||||
"\n",
|
||||
"$$ output = softmax (Z^{o}) = (n_{inputs}, n_{categories}) .$$"
|
||||
"$$ output = softmax (\\hat{z}^{L}) = (n_{inputs}, n_{categories}) .$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1796,7 +1797,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# setup the feed-forward pass\n",
|
||||
"# setup the feed-forward pass, subscript h = hidden layer\n",
|
||||
"\n",
|
||||
"def sigmoid(x):\n",
|
||||
" return 1/(1 + np.exp(-x))\n",
|
||||
@@ -1960,32 +1961,32 @@
|
||||
"To more efficently train our network these equations are implemented using matrix operations. \n",
|
||||
"The error in the output layer is calculated simply as \n",
|
||||
"\n",
|
||||
"$$ \\Delta_o = \\hat{y} - y = (n_{inputs}, n_{categories}) .$$ \n",
|
||||
"$$ \\delta_L = \\hat{y} - y = (n_{inputs}, n_{categories}) .$$ \n",
|
||||
"\n",
|
||||
"The gradient for the output weights is calculated as \n",
|
||||
"\n",
|
||||
"$$ \\nabla W_{o} = A^T \\Delta_o = (n_{hidden}, n_{categories}) ,$$\n",
|
||||
"$$ \\nabla W_{L} = \\hat{a}^T \\delta_L = (n_{hidden}, n_{categories}) ,$$\n",
|
||||
"\n",
|
||||
"where $A = (n_{inputs}, n_{hidden})$. This simply means that we are summing up the gradients for each input. \n",
|
||||
"where $\\hat{a} = (n_{inputs}, n_{hidden})$. This simply means that we are summing up the gradients for each input. \n",
|
||||
"Since we are going backwards we have to transpose the activation matrix. \n",
|
||||
"\n",
|
||||
"The gradient with respect to the output bias is then \n",
|
||||
"\n",
|
||||
"$$ \\nabla B_{o} = \\sum_{i=1}^{n_{inputs}} \\Delta_o = (n_{categories}) .$$ \n",
|
||||
"$$ \\nabla \\hat{b}_{L} = \\sum_{i=1}^{n_{inputs}} \\delta_L = (n_{categories}) .$$ \n",
|
||||
"\n",
|
||||
"The error in the hidden layer is \n",
|
||||
"\n",
|
||||
"$$ \\Delta_h = \\Delta_o W_{o}^T \\circ f'(Z_{h}) = \\Delta_o W_{o}^T \\circ A_{h} \\circ (1 - A_{h}) = (n_{inputs}, n_{hidden}) ,$$ \n",
|
||||
"$$ \\Delta_h = \\delta_L W_{L}^T \\circ f'(z_{h}) = \\delta_L W_{L}^T \\circ a_{h} \\circ (1 - a_{h}) = (n_{inputs}, n_{hidden}) ,$$ \n",
|
||||
"\n",
|
||||
"where $f'(A_{h})$ is the derivative of the activation in the hidden layer. The matrix products mean\n",
|
||||
"where $f'(a_{h})$ is the derivative of the activation in the hidden layer. The matrix products mean\n",
|
||||
"that we are summing up the products for each neuron in the output layer. The symbol $\\circ$ denotes\n",
|
||||
"the *Hadamard product*, meaning element-wise multiplication. \n",
|
||||
"\n",
|
||||
"This again gives us the gradients in the hidden layer: \n",
|
||||
"\n",
|
||||
"$$ \\nabla W_{h} = X^T \\Delta_h = (n_{features}, n_{hidden}) ,$$ \n",
|
||||
"$$ \\nabla W_{h} = X^T \\delta_h = (n_{features}, n_{hidden}) ,$$ \n",
|
||||
"\n",
|
||||
"$$ \\nabla B_{h} = \\sum_{i=1}^{n_{inputs}} \\Delta_h = (n_{hidden}) .$$"
|
||||
"$$ \\nabla b_{h} = \\sum_{i=1}^{n_{inputs}} \\delta_h = (n_{hidden}) .$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -2247,70 +2248,10 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Adjust hyperparameters (if necessary, network architecture\n",
|
||||
"## Adjust hyperparameters\n",
|
||||
"\n",
|
||||
"We now perform a grid search to find the optimal hyperparameters for the network. \n",
|
||||
"Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around $98 \\%$ ($2 \\%$ error rate)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"8\n",
|
||||
" \n",
|
||||
"<\n",
|
||||
"<\n",
|
||||
"<\n",
|
||||
"!\n",
|
||||
"!\n",
|
||||
"C\n",
|
||||
"O\n",
|
||||
"D\n",
|
||||
"E\n",
|
||||
"_\n",
|
||||
"B\n",
|
||||
"L\n",
|
||||
"O\n",
|
||||
"C\n",
|
||||
"K\n",
|
||||
" \n",
|
||||
" \n",
|
||||
"p\n",
|
||||
"y\n",
|
||||
"c\n",
|
||||
"o\n",
|
||||
"d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"9\n",
|
||||
" \n",
|
||||
"<\n",
|
||||
"<\n",
|
||||
"<\n",
|
||||
"!\n",
|
||||
"!\n",
|
||||
"C\n",
|
||||
"O\n",
|
||||
"D\n",
|
||||
"E\n",
|
||||
"_\n",
|
||||
"B\n",
|
||||
"L\n",
|
||||
"O\n",
|
||||
"C\n",
|
||||
"K\n",
|
||||
" \n",
|
||||
" \n",
|
||||
"p\n",
|
||||
"y\n",
|
||||
"c\n",
|
||||
"o\n",
|
||||
"d"
|
||||
"Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around $98\\%$ ($2\\%$ error rate)."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -2321,9 +2262,45 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# optional\n",
|
||||
"eta_vals = np.logspace(-5, 1, 7)\n",
|
||||
"lmbd_vals = np.logspace(-5, 1, 7)\n",
|
||||
"# store the models for later use\n",
|
||||
"DNN_numpy = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)\n",
|
||||
"\n",
|
||||
"# grid search\n",
|
||||
"for i, eta in enumerate(eta_vals):\n",
|
||||
" for j, lmbd in enumerate(lmbd_vals):\n",
|
||||
" dnn = NeuralNetwork(X_train, Y_train_onehot, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size,\n",
|
||||
" n_hidden_neurons=n_hidden_neurons, n_categories=n_categories)\n",
|
||||
" dnn.train()\n",
|
||||
" \n",
|
||||
" DNN_numpy[i][j] = dnn\n",
|
||||
" \n",
|
||||
" test_predict = dnn.predict(X_test)\n",
|
||||
" \n",
|
||||
" print(\"Learning rate = \", eta)\n",
|
||||
" print(\"Lambda = \", lmbd)\n",
|
||||
" print(\"Accuracy score on test set: \", accuracy_score(Y_test, test_predict))\n",
|
||||
" print()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Visualization"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# visual representation of grid search\n",
|
||||
"# uses seaborn heatmap, I believe you can also do this with matplotlib imshow\n",
|
||||
"# uses seaborn heatmap, you can also do this with matplotlib imshow\n",
|
||||
"import seaborn as sns\n",
|
||||
"\n",
|
||||
"sns.set()\n",
|
||||
@@ -2363,45 +2340,55 @@
|
||||
"source": [
|
||||
"## scikit-learn implementation\n",
|
||||
"\n",
|
||||
"scikit-learn is a machine learning library for Python. It focuses more on traditional machine learning methods, such as regression, clustering, decision trees, etc. As such, it has only two types of neural networks: Multi Layer Perceptron outputting continuous values, *MPLRegressor*, and Multi Layer Perceptron outputting labels, *MLPClassifier*. We will see how simple it is to use these classes. \n",
|
||||
"**scikit-learn** focuses more\n",
|
||||
"on traditional machine learning methods, such as regression,\n",
|
||||
"clustering, decision trees, etc. As such, it has only two types of\n",
|
||||
"neural networks: Multi Layer Perceptron outputting continuous values,\n",
|
||||
"*MPLRegressor*, and Multi Layer Perceptron outputting labels,\n",
|
||||
"*MLPClassifier*. We will see how simple it is to use these classes.\n",
|
||||
"\n",
|
||||
"scikit-learn implements a few improvements from our neural network, such as early stopping, a varying learning rate, different optimization methods, etc. We would therefore expect a better performance overall."
|
||||
"**scikit-learn** implements a few improvements from our neural network,\n",
|
||||
"such as early stopping, a varying learning rate, different\n",
|
||||
"optimization methods, etc. We would therefore expect a better\n",
|
||||
"performance overall."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.neural_network import MLPClassifier\n",
|
||||
"# store models for later use\n",
|
||||
"DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)\n",
|
||||
"\n",
|
||||
"for i, eta in enumerate(eta_vals):\n",
|
||||
" for j, lmbd in enumerate(lmbd_vals):\n",
|
||||
" dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',\n",
|
||||
" alpha=lmbd, learning_rate_init=eta, max_iter=epochs)\n",
|
||||
" dnn.fit(X_train, Y_train)\n",
|
||||
" \n",
|
||||
" DNN_scikit[i][j] = dnn\n",
|
||||
" \n",
|
||||
" print(\"Learning rate = \", eta)\n",
|
||||
" print(\"Lambda = \", lmbd)\n",
|
||||
" print(\"Accuracy score on test set: \", dnn.score(X_test, Y_test))\n",
|
||||
" print()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"1\n",
|
||||
"1\n",
|
||||
" \n",
|
||||
"<\n",
|
||||
"<\n",
|
||||
"<\n",
|
||||
"!\n",
|
||||
"!\n",
|
||||
"C\n",
|
||||
"O\n",
|
||||
"D\n",
|
||||
"E\n",
|
||||
"_\n",
|
||||
"B\n",
|
||||
"L\n",
|
||||
"O\n",
|
||||
"C\n",
|
||||
"K\n",
|
||||
" \n",
|
||||
" \n",
|
||||
"p\n",
|
||||
"y\n",
|
||||
"c\n",
|
||||
"o\n",
|
||||
"d"
|
||||
"## Visualization"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"execution_count": 12,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2486,7 +2473,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"execution_count": 13,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2504,7 +2491,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"execution_count": 14,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2522,7 +2509,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"execution_count": 15,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2574,7 +2561,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"execution_count": 16,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2606,7 +2593,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"execution_count": 17,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2754,7 +2741,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"execution_count": 18,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2765,14 +2752,13 @@
|
||||
"n_neurons_layer1 = 100\n",
|
||||
"n_neurons_layer2 = 50\n",
|
||||
"n_categories = 10\n",
|
||||
"\n",
|
||||
"eta_vals = np.logspace(-5, 1, 7)\n",
|
||||
"lmbd_vals = np.logspace(-5, 1, 7)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"execution_count": 19,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2797,7 +2783,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"execution_count": 20,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2838,7 +2824,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"execution_count": 21,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2864,7 +2850,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 20,
|
||||
"execution_count": 22,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2882,7 +2868,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 21,
|
||||
"execution_count": 23,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2900,7 +2886,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 22,
|
||||
"execution_count": 24,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2925,7 +2911,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 23,
|
||||
"execution_count": 25,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2950,7 +2936,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 24,
|
||||
"execution_count": 26,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -192,7 +192,7 @@ The output $y$ is produced via the activation function $f$
|
||||
\]
|
||||
!et
|
||||
This function receives $x_i$ as inputs.
|
||||
Here the activation $z=\sum_{i=1}^n w_ix_i$.
|
||||
Here the activation $z=(\sum_{i=1}^n w_ix_i+b_i)$.
|
||||
In an FFNN of such neurons, the *inputs* $x_i$ are the *outputs* of
|
||||
the neurons in the preceding layer. Furthermore, an MLP is
|
||||
fully-connected, which means that each neuron receives a weighted sum
|
||||
@@ -312,7 +312,7 @@ is the bias $b_i^l$ and activation $y_i^l$ of node $i$ in layer $l$ respectively
|
||||
|
||||
We have that $\mathrm{W}_l$ is an $N_{l-1} \times N_l$ matrix, while $\hat{b}_l$ and $\hat{y}_l$ are $N_l \times 1$ column vectors.
|
||||
With this notation, the sum becomes a matrix-vector multiplication, and we can write
|
||||
the equation for the activations of hidden layer 2 as
|
||||
the equation for the activations of hidden layer 2 (assuming three nodes for simplicity) as
|
||||
!bt
|
||||
\begin{equation}
|
||||
\hat{y}_2 = f_2(\mathrm{W}_2 \hat{y}_{1} + \hat{b}_{2}) =
|
||||
@@ -489,7 +489,7 @@ o The input nodes pass values to the first hidden layer, its nodes pass the info
|
||||
As a convention it is normal to call a network with one layer of input units, one layer of hidden
|
||||
units and one layer of output units as a two-layer network. A network with two layers of hidden units is called a three-layer network etc etc.
|
||||
|
||||
For an MLP there is no direct connection between the output nodes/neurons/units and the input nodes/neurons/units.
|
||||
For an MLP network there is no direct connection between the output nodes/neurons/units and the input nodes/neurons/units.
|
||||
Hereafter we will call the various entities of a layer for nodes.
|
||||
There are also no connections within a single layer.
|
||||
|
||||
@@ -591,11 +591,10 @@ compact form as the matrix-vector products we discussed earlier,
|
||||
\]
|
||||
!et
|
||||
|
||||
With the activation function $\hat{z}^l$ we can in turn define the
|
||||
With the activation values $\hat{z}^l$ we can in turn define the
|
||||
output of layer $l$ as $\hat{a}^l = f(\hat{z}^l)$ where $f$ is our
|
||||
activation function. In the examples here we will use the sigmoid
|
||||
function discussed in our logistic regression lectures and here as
|
||||
well. We will also use the same activation function $f$ for all layers
|
||||
function discussed in our logistic regression lectures. We will also use the same activation function $f$ for all layers
|
||||
and their nodes. It means we have
|
||||
|
||||
!bt
|
||||
@@ -680,7 +679,7 @@ and using the Hadamard product of two vectors we can write this as
|
||||
!et
|
||||
|
||||
This is an important expression. The second term on the right handside
|
||||
measures how fast the cost is changing as a function of the $j$th
|
||||
measures how fast the cost function is changing as a function of the $j$th
|
||||
output activation. If, for example, the cost function doesn't depend
|
||||
much on a particular output node $j$, then $\delta_j^L$ will be small,
|
||||
which is what we would expect. The first term on the right, measures
|
||||
@@ -691,9 +690,9 @@ Notice that everything in the above equations is easily computed. In
|
||||
particular, we compute $z_j^L$ while computing the behaviour of the
|
||||
network, and it is only a small additional overhead to compute
|
||||
$f'(z^L_j)$. The exact form of the derivative with respect to the
|
||||
outpuwill, of course, depend on the form of the cost function.
|
||||
output depends on the form of the cost function.
|
||||
However, provided the cost function is known there should be little
|
||||
trouble computing
|
||||
trouble in calculating
|
||||
|
||||
!bt
|
||||
\[
|
||||
@@ -729,6 +728,7 @@ That is, the error $\delta_j^L$ is exactly equal to the rate of change of the co
|
||||
===== Bringing it together =====
|
||||
|
||||
We have now three equations that are essential for the computations of the derivatives of the cost function at the output layer. These equations are needed to start the algorithm and they are
|
||||
|
||||
!bblock The starting equations
|
||||
|
||||
!bt
|
||||
@@ -760,7 +760,7 @@ that it changes slowly when we minimize the weights via say gradient
|
||||
descent. In this case we say the system learns slowly.
|
||||
|
||||
Another interesting feature is that is when the activation function,
|
||||
represented by the sigmoid function here, is rather flat when towards
|
||||
represented by the sigmoid function here, is rather flat when we move towards
|
||||
its end values $0$ and $1$ (see the above Python codes). In these
|
||||
cases, the derivatives of the activation function will also be close
|
||||
to zero, meaning again that the gradients will be small and the
|
||||
@@ -859,7 +859,7 @@ b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^L},
|
||||
!eblock
|
||||
|
||||
The parameter $\eta$ is the learning parameter discussed in connection with the gradient descent methods.
|
||||
Here it is convenient to use stochastic radient descent with mini-batches with an outer loop that steps through multiple epochs of training.
|
||||
Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
|
||||
|
||||
|
||||
!split
|
||||
@@ -880,11 +880,11 @@ also has other nice properties, such as a derivative that is simple to
|
||||
calculate.
|
||||
|
||||
For an input $\boldsymbol{a}$ from the hidden layer, the probability that the input $\boldsymbol{x}$
|
||||
is in class 0 or 1 is just:
|
||||
|
||||
is in class 0 or 1 is just. We let $\theta$ represent the unknown weights and biases to be adjusted by our equations). The variable $x$
|
||||
represents our activation values $z$. We have
|
||||
!bt
|
||||
\[
|
||||
P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) = \frac{1}{1 + \exp (- \boldsymbol{a}^T \boldsymbol{w}_{out})} ,
|
||||
P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) = \frac{1}{1 + \exp (- \boldsymbol{x}} ,
|
||||
\]
|
||||
!et
|
||||
and
|
||||
@@ -1165,7 +1165,7 @@ i.e. each neuron $j$ outputs the probability of being in class $j$ given an inpu
|
||||
The denominator is a normalization factor to ensure the outputs (probabilities) sum up to 1.
|
||||
The exponent is just the weighted sum of inputs as before:
|
||||
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i = \boldsymbol{a}^T \boldsymbol{w}_j .$$
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i+b_j.$$
|
||||
|
||||
Since each neuron in the output layer is connected to the 50 inputs from the hidden layer we have 50x10 = 500
|
||||
weights to the output layer.
|
||||
@@ -1179,7 +1179,7 @@ or normal distribution. Setting all weights to zero means all neurons give the s
|
||||
Adding a bias value to the weighted sum of inputs allows the neural network to represent a greater range
|
||||
of values. Without it, any input with the value 0 will be mapped to zero (before being passed through the activation). The bias unit has an output of 1, and a weight to each neuron $j$, $b_j$:
|
||||
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i + 1\cdot b_j = \boldsymbol{a}^T \boldsymbol{w}_j + b_j .$$
|
||||
$$ z_j = \sum_{i=1}^n w_ {ij} a_i + 1\cdot b_j.$$
|
||||
|
||||
The bias weights $\boldsymbol{b}$ are often initialized to zero, but a small value like $0.01$ ensures all neurons have some output which can be backpropagated in the first training cycle.
|
||||
!bc pycod
|
||||
@@ -1204,22 +1204,22 @@ output_bias = np.zeros(n_categories) + 0.01
|
||||
===== Feed-forward pass =====
|
||||
|
||||
Denote $F$ the number of features, $H$ the number of hidden neurons and $C$ the number of categories.
|
||||
For each input image we calculate a weighted sum of input features (pixel values) to each neuron $j$ in the hidden layer:
|
||||
For each input image we calculate a weighted sum of input features (pixel values) to each neuron $j$ in the hidden layer $l$:
|
||||
|
||||
$$ z_{j}^{h} = \sum_{i=1}^{F} w_{ij}^{h} x_i + b_{j}^{h} = \boldsymbol{x}^T \boldsymbol{w}_{j}^{h} + b_{j}^{h} ,$$
|
||||
$$ z_{j}^{l} = \sum_{i=1}^{F} w_{ij}^{l} x_i + b_{j}^{l},$$
|
||||
|
||||
this is then passed through our activation function
|
||||
|
||||
$$ a_{j}^{h} = f(z_{j}^{h}) .$$
|
||||
$$ a_{j}^{l} = f(z_{j}^{l}) .$$
|
||||
|
||||
We calculate a weighted sum of inputs (activations in the hidden layer) to each neuron $j$ in the output layer:
|
||||
|
||||
$$ z_{j}^{o} = \sum_{i=1}^{H} w_{ij}^{o} a_{i}^{h} + b_{j}^{o} = (\boldsymbol{a}^{h})^T \boldsymbol{w}_{j}^{o} + b_{j}^{o} .$$
|
||||
$$ z_{j}^{L} = \sum_{i=1}^{H} w_{ij}^{L} a_{i}^{l} + b_{j}^{L}.$$
|
||||
|
||||
Finally we calculate the output of neuron $j$ in the output layer using the softmax function:
|
||||
|
||||
$$ a_{j}^{o} = \frac{\exp{(z_j^{o})}}
|
||||
{\sum_{c=0}^{C-1} \exp{(z_c^{o})}} .$$
|
||||
$$ a_{j}^{L} = \frac{\exp{(z_j^{L})}}
|
||||
{\sum_{c=0}^{C-1} \exp{(z_c^{L})}} .$$
|
||||
|
||||
!split
|
||||
===== Matrix multiplication =====
|
||||
@@ -1235,24 +1235,24 @@ and obtain a matrix that holds the weighted sum of inputs to the hidden layer
|
||||
for each input image and each hidden neuron.
|
||||
We also add the bias to obtain a matrix of weighted sums to the hidden layer $Z^{h}$:
|
||||
|
||||
$$ Z^{h} = X W^{h} + B^{h} ,$$
|
||||
$$ \hat{z}^{l} = \hat{X} \hat{W}^{l} + \hat{b}^{l} ,$$
|
||||
|
||||
meaning the same bias (1D array with size equal number of hidden neurons) is added to each input image.
|
||||
This is then passed through the activation:
|
||||
|
||||
$$ A^{h} = f(Z^h) .$$
|
||||
$$ \hat{a}^{l} = f(\hat{z}^l) .$$
|
||||
|
||||
This is fed to the output layer:
|
||||
|
||||
$$ Z^{o} = A^{h} W^{o} + B^{o} .$$
|
||||
$$ \hat{z}^{L} = \hat{a}^{L} \hat{W}^{L} + \hat{b}^{L} .$$
|
||||
|
||||
Finally we receive our output values for each image and each category by passing it through the softmax function:
|
||||
|
||||
$$ output = softmax (Z^{o}) = (n_{inputs}, n_{categories}) .$$
|
||||
$$ output = softmax (\hat{z}^{L}) = (n_{inputs}, n_{categories}) .$$
|
||||
|
||||
|
||||
!bc pycod
|
||||
# setup the feed-forward pass
|
||||
# setup the feed-forward pass, subscript h = hidden layer
|
||||
|
||||
def sigmoid(x):
|
||||
return 1/(1 + np.exp(-x))
|
||||
@@ -1386,32 +1386,32 @@ calculate the gradient efficently.
|
||||
To more efficently train our network these equations are implemented using matrix operations.
|
||||
The error in the output layer is calculated simply as
|
||||
|
||||
$$ \Delta_o = \hat{y} - y = (n_{inputs}, n_{categories}) .$$
|
||||
$$ \delta_L = \hat{y} - y = (n_{inputs}, n_{categories}) .$$
|
||||
|
||||
The gradient for the output weights is calculated as
|
||||
|
||||
$$ \nabla W_{o} = A^T \Delta_o = (n_{hidden}, n_{categories}) ,$$
|
||||
$$ \nabla W_{L} = \hat{a}^T \delta_L = (n_{hidden}, n_{categories}) ,$$
|
||||
|
||||
where $A = (n_{inputs}, n_{hidden})$. This simply means that we are summing up the gradients for each input.
|
||||
where $\hat{a} = (n_{inputs}, n_{hidden})$. This simply means that we are summing up the gradients for each input.
|
||||
Since we are going backwards we have to transpose the activation matrix.
|
||||
|
||||
The gradient with respect to the output bias is then
|
||||
|
||||
$$ \nabla B_{o} = \sum_{i=1}^{n_{inputs}} \Delta_o = (n_{categories}) .$$
|
||||
$$ \nabla \hat{b}_{L} = \sum_{i=1}^{n_{inputs}} \delta_L = (n_{categories}) .$$
|
||||
|
||||
The error in the hidden layer is
|
||||
|
||||
$$ \Delta_h = \Delta_o W_{o}^T \circ f'(Z_{h}) = \Delta_o W_{o}^T \circ A_{h} \circ (1 - A_{h}) = (n_{inputs}, n_{hidden}) ,$$
|
||||
$$ \Delta_h = \delta_L W_{L}^T \circ f'(z_{h}) = \delta_L W_{L}^T \circ a_{h} \circ (1 - a_{h}) = (n_{inputs}, n_{hidden}) ,$$
|
||||
|
||||
where $f'(A_{h})$ is the derivative of the activation in the hidden layer. The matrix products mean
|
||||
where $f'(a_{h})$ is the derivative of the activation in the hidden layer. The matrix products mean
|
||||
that we are summing up the products for each neuron in the output layer. The symbol $\circ$ denotes
|
||||
the *Hadamard product*, meaning element-wise multiplication.
|
||||
|
||||
This again gives us the gradients in the hidden layer:
|
||||
|
||||
$$ \nabla W_{h} = X^T \Delta_h = (n_{features}, n_{hidden}) ,$$
|
||||
$$ \nabla W_{h} = X^T \delta_h = (n_{features}, n_{hidden}) ,$$
|
||||
|
||||
$$ \nabla B_{h} = \sum_{i=1}^{n_{inputs}} \Delta_h = (n_{hidden}) .$$
|
||||
$$ \nabla b_{h} = \sum_{i=1}^{n_{inputs}} \delta_h = (n_{hidden}) .$$
|
||||
|
||||
|
||||
!bc pycod
|
||||
@@ -1644,17 +1644,14 @@ def accuracy_score_numpy(Y_test, Y_pred):
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Adjust hyperparameters (if necessary, network architecture =====
|
||||
===== Adjust hyperparameters =====
|
||||
|
||||
We now perform a grid search to find the optimal hyperparameters for the network.
|
||||
Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around $98 \%$ ($2 \%$ error rate).
|
||||
|
||||
Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around $98\%$ ($2\%$ error rate).
|
||||
|
||||
!bc pycod
|
||||
eta_vals = np.logspace(-5, 1, 7)
|
||||
lmbd_vals = np.logspace(-5, 1, 7)
|
||||
!ec
|
||||
!bc pycod
|
||||
# store the models for later use
|
||||
DNN_numpy = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
|
||||
|
||||
@@ -1674,10 +1671,13 @@ for i, eta in enumerate(eta_vals):
|
||||
print("Accuracy score on test set: ", accuracy_score(Y_test, test_predict))
|
||||
print()
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Visualization =====
|
||||
|
||||
!bc pycod
|
||||
# optional
|
||||
# visual representation of grid search
|
||||
# uses seaborn heatmap, I believe you can also do this with matplotlib imshow
|
||||
# uses seaborn heatmap, you can also do this with matplotlib imshow
|
||||
import seaborn as sns
|
||||
|
||||
sns.set()
|
||||
@@ -1714,14 +1714,20 @@ plt.show()
|
||||
!split
|
||||
===== scikit-learn implementation =====
|
||||
|
||||
scikit-learn is a machine learning library for Python. It focuses more on traditional machine learning methods, such as regression, clustering, decision trees, etc. As such, it has only two types of neural networks: Multi Layer Perceptron outputting continuous values, *MPLRegressor*, and Multi Layer Perceptron outputting labels, *MLPClassifier*. We will see how simple it is to use these classes.
|
||||
_scikit-learn_ focuses more
|
||||
on traditional machine learning methods, such as regression,
|
||||
clustering, decision trees, etc. As such, it has only two types of
|
||||
neural networks: Multi Layer Perceptron outputting continuous values,
|
||||
*MPLRegressor*, and Multi Layer Perceptron outputting labels,
|
||||
*MLPClassifier*. We will see how simple it is to use these classes.
|
||||
|
||||
scikit-learn implements a few improvements from our neural network, such as early stopping, a varying learning rate, different optimization methods, etc. We would therefore expect a better performance overall.
|
||||
|
||||
_scikit-learn_ implements a few improvements from our neural network,
|
||||
such as early stopping, a varying learning rate, different
|
||||
optimization methods, etc. We would therefore expect a better
|
||||
performance overall.
|
||||
|
||||
!bc pycod
|
||||
from sklearn.neural_network import MLPClassifier
|
||||
|
||||
# store models for later use
|
||||
DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
|
||||
|
||||
@@ -1738,6 +1744,10 @@ for i, eta in enumerate(eta_vals):
|
||||
print("Accuracy score on test set: ", dnn.score(X_test, Y_test))
|
||||
print()
|
||||
!ec
|
||||
|
||||
|
||||
!split
|
||||
===== Visualization =====
|
||||
!bc pycod
|
||||
# optional
|
||||
# visual representation of grid search
|
||||
@@ -2033,7 +2043,6 @@ batch_size = 100
|
||||
n_neurons_layer1 = 100
|
||||
n_neurons_layer2 = 50
|
||||
n_categories = 10
|
||||
|
||||
eta_vals = np.logspace(-5, 1, 7)
|
||||
lmbd_vals = np.logspace(-5, 1, 7)
|
||||
!ec
|
||||
@@ -2055,7 +2064,6 @@ for i, eta in enumerate(eta_vals):
|
||||
print("Lambda = ", lmbd)
|
||||
print("Test accuracy: %.3f" % DNN.test_accuracy)
|
||||
print()
|
||||
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
|
||||
Reference in New Issue
Block a user