397 lines
19 KiB
HTML
397 lines
19 KiB
HTML
<!--
|
|
Automatically generated HTML file from DocOnce source
|
|
(https://github.com/hplgit/doconce/)
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
|
<meta name="description" content="Data Analysis and Machine Learning: Using Neural networks to solve ODEs and PDEs">
|
|
|
|
<title>Data Analysis and Machine Learning: Using Neural networks to solve ODEs and PDEs</title>
|
|
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
|
|
<style type="text/css">
|
|
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
|
|
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Differential equations', 2, None, '___sec0'),
|
|
('Description of the equation to solve for', 2, None, '___sec1'),
|
|
('Ordinary Differential Equations', 2, None, '___sec2'),
|
|
('The trial solution', 2, None, '___sec3'),
|
|
('Minimizing the cost function using gradient descent and '
|
|
'automatic differentiation',
|
|
2,
|
|
None,
|
|
'___sec4'),
|
|
('Example: Exponential decay and setting up the network using '
|
|
'Autograd',
|
|
2,
|
|
None,
|
|
'___sec5'),
|
|
('The function to solve for', 2, None, '___sec6'),
|
|
('The trial solution', 2, None, '___sec7'),
|
|
('Reformulating the problem', 2, None, '___sec8'),
|
|
('A possible implementation of a neural network using Autograd',
|
|
2,
|
|
None,
|
|
'___sec9'),
|
|
('Backpropagation using Autograd', 2, None, '___sec10'),
|
|
('Gradient descent', 2, None, '___sec11'),
|
|
('The network with one input, hidden, and output layer',
|
|
2,
|
|
None,
|
|
'___sec12'),
|
|
('The network with one input layer, specified number of hidden '
|
|
'layers, and one output layer output layer',
|
|
2,
|
|
None,
|
|
'___sec13'),
|
|
('Example: Population growth, comparing Autograd, TensorFlow, '
|
|
"and Euler's scheme",
|
|
2,
|
|
None,
|
|
'___sec14'),
|
|
('Setting up the problem', 2, None, '___sec15'),
|
|
('The trial solution', 2, None, '___sec16'),
|
|
('The program using Autograd', 2, None, '___sec17'),
|
|
('Using forward Euler to solve the ODE', 2, None, '___sec18'),
|
|
('Using TensorFlow to model logistic population growth',
|
|
2,
|
|
None,
|
|
'___sec19'),
|
|
('The general program flow in TensorFlow', 2, None, '___sec20'),
|
|
('Program flow in TensorFlow - Construction phase',
|
|
2,
|
|
None,
|
|
'___sec21'),
|
|
('Program flow in TensorFlow - Execution phase',
|
|
2,
|
|
None,
|
|
'___sec22'),
|
|
('The full program modeling logistic population growth using '
|
|
'TensorFlow',
|
|
2,
|
|
None,
|
|
'___sec23'),
|
|
('Example: Solving the one dimensional Poisson equation using '
|
|
'Autograd and TensorFlow',
|
|
2,
|
|
None,
|
|
'___sec24'),
|
|
('The specific equation to solve for', 2, None, '___sec25'),
|
|
('Solving the equation using Autograd', 2, None, '___sec26'),
|
|
('Comparing with a numerical scheme', 2, None, '___sec27'),
|
|
('Using gradient descent in TensorFlow to solve Poisson equation',
|
|
2,
|
|
None,
|
|
'___sec28'),
|
|
('Using a different optimization algorithm implemented in '
|
|
'TensorFlow to solve Poisson equation',
|
|
2,
|
|
None,
|
|
'___sec29'),
|
|
('Partial Differential Equations', 2, None, '___sec30'),
|
|
('Example: The diffusion equation', 2, None, '___sec31'),
|
|
('Defining the problem', 2, None, '___sec32'),
|
|
('Setting up the network using Autograd', 2, None, '___sec33'),
|
|
('Setting up the network using Autograd; The trial solution',
|
|
2,
|
|
None,
|
|
'___sec34'),
|
|
('Setting up the network using Autograd; The full program',
|
|
2,
|
|
None,
|
|
'___sec35'),
|
|
('Example: Solving the wave equation using Autograd and '
|
|
'TensorFlow',
|
|
2,
|
|
None,
|
|
'___sec36'),
|
|
('The problem to solve for', 2, None, '___sec37'),
|
|
('The trial solution', 2, None, '___sec38'),
|
|
('The analytical solution', 2, None, '___sec39'),
|
|
('Solving the wave equation - the full program using Autograd',
|
|
2,
|
|
None,
|
|
'___sec40'),
|
|
('Solving the wave equation - the full program using TensorFlow',
|
|
2,
|
|
None,
|
|
'___sec41'),
|
|
('Resources', 2, None, '___sec42')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="odenn-bs.html">Data Analysis and Machine Learning: Using Neural networks to solve ODEs and PDEs</a>
|
|
</div>
|
|
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs001.html#___sec0" style="font-size: 80%;">Differential equations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs002.html#___sec1" style="font-size: 80%;">Description of the equation to solve for</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs003.html#___sec2" style="font-size: 80%;">Ordinary Differential Equations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs004.html#___sec3" style="font-size: 80%;">The trial solution</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs005.html#___sec4" style="font-size: 80%;">Minimizing the cost function using gradient descent and automatic differentiation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs006.html#___sec5" style="font-size: 80%;">Example: Exponential decay and setting up the network using Autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs007.html#___sec6" style="font-size: 80%;">The function to solve for</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs008.html#___sec7" style="font-size: 80%;">The trial solution</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs009.html#___sec8" style="font-size: 80%;">Reformulating the problem</a></li>
|
|
<!-- navigation toc: --> <li><a href="#___sec9" style="font-size: 80%;">A possible implementation of a neural network using Autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs011.html#___sec10" style="font-size: 80%;">Backpropagation using Autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs012.html#___sec11" style="font-size: 80%;">Gradient descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs013.html#___sec12" style="font-size: 80%;">The network with one input, hidden, and output layer</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs014.html#___sec13" style="font-size: 80%;">The network with one input layer, specified number of hidden layers, and one output layer output layer</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs015.html#___sec14" style="font-size: 80%;">Example: Population growth, comparing Autograd, TensorFlow, and Euler's scheme</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs016.html#___sec15" style="font-size: 80%;">Setting up the problem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs017.html#___sec16" style="font-size: 80%;">The trial solution</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs018.html#___sec17" style="font-size: 80%;">The program using Autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs019.html#___sec18" style="font-size: 80%;">Using forward Euler to solve the ODE</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs020.html#___sec19" style="font-size: 80%;">Using TensorFlow to model logistic population growth</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs021.html#___sec20" style="font-size: 80%;">The general program flow in TensorFlow</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs021.html#___sec21" style="font-size: 80%;">Program flow in TensorFlow - Construction phase</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs021.html#___sec22" style="font-size: 80%;">Program flow in TensorFlow - Execution phase</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs021.html#___sec23" style="font-size: 80%;">The full program modeling logistic population growth using TensorFlow</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs022.html#___sec24" style="font-size: 80%;">Example: Solving the one dimensional Poisson equation using Autograd and TensorFlow</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs023.html#___sec25" style="font-size: 80%;">The specific equation to solve for</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs024.html#___sec26" style="font-size: 80%;">Solving the equation using Autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs025.html#___sec27" style="font-size: 80%;">Comparing with a numerical scheme</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs026.html#___sec28" style="font-size: 80%;">Using gradient descent in TensorFlow to solve Poisson equation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs027.html#___sec29" style="font-size: 80%;">Using a different optimization algorithm implemented in TensorFlow to solve Poisson equation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs028.html#___sec30" style="font-size: 80%;">Partial Differential Equations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs029.html#___sec31" style="font-size: 80%;">Example: The diffusion equation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs030.html#___sec32" style="font-size: 80%;">Defining the problem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs031.html#___sec33" style="font-size: 80%;">Setting up the network using Autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs032.html#___sec34" style="font-size: 80%;">Setting up the network using Autograd; The trial solution</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs033.html#___sec35" style="font-size: 80%;">Setting up the network using Autograd; The full program</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs034.html#___sec36" style="font-size: 80%;">Example: Solving the wave equation using Autograd and TensorFlow</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs035.html#___sec37" style="font-size: 80%;">The problem to solve for</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs036.html#___sec38" style="font-size: 80%;">The trial solution</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs037.html#___sec39" style="font-size: 80%;">The analytical solution</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs038.html#___sec40" style="font-size: 80%;">Solving the wave equation - the full program using Autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs039.html#___sec41" style="font-size: 80%;">Solving the wave equation - the full program using TensorFlow</a></li>
|
|
<!-- navigation toc: --> <li><a href="._odenn-bs040.html#___sec42" style="font-size: 80%;">Resources</a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
|
|
<div class="container">
|
|
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
|
|
<a name="part0010"></a>
|
|
<!-- !split -->
|
|
|
|
<h2 id="___sec9" class="anchor">A possible implementation of a neural network using Autograd </h2>
|
|
|
|
<p>
|
|
For simplicity, it is assumed that the input is an array \( \vec x = (x_1, \dots, x_N) \) with \( N \) elements. It is at these points the neural network should find \( P \) such that it fulfills <a href="._odenn-bs009.html#mjx-eqn-9">(9)</a>.
|
|
|
|
<p>
|
|
First, the neural network must feed forward the inputs.
|
|
This means that \( \vec x \) must be passed through an input layer, a hidden layer and a output layer. The input layer in this case, does not need to process the data any further.
|
|
The input layer will consist of \( N_{\text{input} } \) neurons, passing its element to each neuron in the hidden layer. The number of neurons in the hidden layer will be \( N_{\text{hidden} } \).
|
|
|
|
<p>
|
|
For the \( i \)-th in the hidden layer with weight \( w_i^{\text{hidden} } \) and bias \( b_i^{\text{hidden} } \), the weighting from the \( j \)-th neuron at the input layer is:
|
|
|
|
$$
|
|
\begin{aligned}
|
|
z_{i,j}^{\text{hidden}} &= b_i^{\text{hidden}} + w_i^{\text{hidden}}x_j \\
|
|
&=
|
|
\begin{pmatrix}
|
|
b_i^{\text{hidden}} & w_i^{\text{hidden}}
|
|
\end{pmatrix}
|
|
\begin{pmatrix}
|
|
1 \\
|
|
x_j
|
|
\end{pmatrix}
|
|
\end{aligned}
|
|
$$
|
|
|
|
<p>
|
|
The result after weighting the inputs at the \( i \)-th hidden neuron can be written as a vector:
|
|
|
|
$$
|
|
\begin{aligned}
|
|
\vec{z}_{i}^{\text{hidden}} &= \Big( b_i^{\text{hidden}} + w_i^{\text{hidden}}x_1 , \ b_i^{\text{hidden}} + w_i^{\text{hidden}} x_2, \ \dots \, , \ b_i^{\text{hidden}} + w_i^{\text{hidden}} x_N\Big) \\
|
|
&=
|
|
\begin{pmatrix}
|
|
b_i^{\text{hidden}} & w_i^{\text{hidden}}
|
|
\end{pmatrix}
|
|
\begin{pmatrix}
|
|
1 & 1 & \dots & 1 \\
|
|
x_1 & x_2 & \dots & x_N
|
|
\end{pmatrix} \\
|
|
&= \vec{p}_{i, \text{hidden}}^T X
|
|
\end{aligned}
|
|
$$
|
|
|
|
<p>
|
|
The vector \( \vec{p}_{i, \text{hidden}}^T \) constitutes each row in \( P_{\text{hidden} } \), which contains the weights for the neural network to minimize according to <a href="._odenn-bs009.html#mjx-eqn-9">(9)</a>.
|
|
|
|
<p>
|
|
After having found \( \vec{z}_{i}^{\text{hidden}} \) for every \( i \)-th neuron within the hidden layer, the vector will be sent to an activation function \( a_i(\vec{z}) \).
|
|
|
|
<p>
|
|
In this example, the sigmoid function has been chosen to be the activation function for each hidden neuron:
|
|
|
|
$$
|
|
f(z) = \frac{1}{1 + \exp{(-z)}}
|
|
$$
|
|
|
|
<p>
|
|
It is possible to use other activations functions for the hidden layer also.
|
|
|
|
<p>
|
|
The output $\vec{x}_i^{\text{hidden} }$from each \( i \)-th hidden neuron is:
|
|
|
|
$$
|
|
\vec{x}_i^{\text{hidden} } = f\big( \vec{z}_{i}^{\text{hidden}} \big)
|
|
$$
|
|
|
|
<p>
|
|
The outputs \( \vec{x}_i^{\text{hidden} } \) are then sent to the output layer.
|
|
|
|
<p>
|
|
The output layer consists of one neuron in this case, and combines the output from each of the neurons in the hidden layers. The output layer combines the results from the hidden layer using some weights $ w_i^{\text{output}}$ and biases \( b_i^{\text{output}} \). In this case, it is assumes that the number of neurons in the output layer is one.
|
|
|
|
<p>
|
|
The procedure of weighting the output neuron \( j \) in the hidden layer to the \( i \)-th neuron in the output layer is similar as for the hidden layer described previously.
|
|
|
|
$$
|
|
\begin{aligned}
|
|
z_{1,j}^{\text{output}} & =
|
|
\begin{pmatrix}
|
|
b_1^{\text{output}} & \vec{w}_1^{\text{output}}
|
|
\end{pmatrix}
|
|
\begin{pmatrix}
|
|
1 \\
|
|
\vec{x}_j^{\text{hidden}}
|
|
\end{pmatrix}
|
|
\end{aligned}
|
|
$$
|
|
|
|
<p>
|
|
Expressing \( z_{1,j}^{\text{output}} \) as a vector gives the following way of weighting the inputs from the hidden layer:
|
|
|
|
$$
|
|
\vec{z}_{1}^{\text{output}} =
|
|
\begin{pmatrix}
|
|
b_1^{\text{output}} & \vec{w}_1^{\text{output}}
|
|
\end{pmatrix}
|
|
\begin{pmatrix}
|
|
1 & 1 & \dots & 1 \\
|
|
\vec{x}_1^{\text{hidden}} & \vec{x}_2^{\text{hidden}} & \dots & \vec{x}_N^{\text{hidden}}
|
|
\end{pmatrix}
|
|
$$
|
|
|
|
<p>
|
|
In this case we seek a continuous range of values since we are approximating a function. This means that after computing \( \vec{z}_{1}^{\text{output}} \) the neural network has finished its feed forward step, and \( \vec{z}_{1}^{\text{output}} \) is the final output of the network.
|
|
|
|
<p>
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._odenn-bs009.html">«</a></li>
|
|
<li><a href="._odenn-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._odenn-bs002.html">3</a></li>
|
|
<li><a href="._odenn-bs003.html">4</a></li>
|
|
<li><a href="._odenn-bs004.html">5</a></li>
|
|
<li><a href="._odenn-bs005.html">6</a></li>
|
|
<li><a href="._odenn-bs006.html">7</a></li>
|
|
<li><a href="._odenn-bs007.html">8</a></li>
|
|
<li><a href="._odenn-bs008.html">9</a></li>
|
|
<li><a href="._odenn-bs009.html">10</a></li>
|
|
<li class="active"><a href="._odenn-bs010.html">11</a></li>
|
|
<li><a href="._odenn-bs011.html">12</a></li>
|
|
<li><a href="._odenn-bs012.html">13</a></li>
|
|
<li><a href="._odenn-bs013.html">14</a></li>
|
|
<li><a href="._odenn-bs014.html">15</a></li>
|
|
<li><a href="._odenn-bs015.html">16</a></li>
|
|
<li><a href="._odenn-bs016.html">17</a></li>
|
|
<li><a href="._odenn-bs017.html">18</a></li>
|
|
<li><a href="._odenn-bs018.html">19</a></li>
|
|
<li><a href="._odenn-bs019.html">20</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._odenn-bs040.html">41</a></li>
|
|
<li><a href="._odenn-bs011.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
|
</footer>
|
|
-->
|
|
|
|
|
|
</body>
|
|
</html>
|
|
|
|
|