mhjensen
314 lines
21 KiB
HTML
314 lines
21 KiB
HTML
<!--
|
|
Automatically generated HTML file from DocOnce source
|
|
(https://github.com/hplgit/doconce/)
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Data Analysis and Machine Learning: Elements of Bayesian theory and Bayesian Neural Networks">
|
|
|
|
<title>Data Analysis and Machine Learning: Elements of Bayesian theory and Bayesian Neural Networks</title>
|
|
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
|
|
<style type="text/css">
|
|
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
|
|
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Why Bayesian Statistics?', 2, None, '___sec0'),
|
|
('Inference', 2, None, '___sec1'),
|
|
('Statistical Inference', 2, None, '___sec2'),
|
|
('Some history', 2, None, '___sec3'),
|
|
('The Bayesian recipe', 2, None, '___sec4'),
|
|
("Bayes' theorem", 2, None, '___sec5'),
|
|
("The friends of Bayes' theorem", 2, None, '___sec6'),
|
|
('Inference With Parametric Models', 2, None, '___sec7'),
|
|
('Illustrative examples with python code', 2, None, '___sec8'),
|
|
('Example: Is this a fair coin?', 2, None, '___sec9'),
|
|
('A few words on different priors', 2, None, '___sec10'),
|
|
('Bayesian parameter estimation (single parameter)',
|
|
2,
|
|
None,
|
|
'___sec11'),
|
|
('Example: Measured flux from a star', 3, None, '___sec12'),
|
|
('Simple Photon Counts: Frequentist Approach',
|
|
3,
|
|
None,
|
|
'___sec13'),
|
|
('Simple Photon Counts: Bayesian Approach', 3, None, '___sec14'),
|
|
('A note about priors', 3, None, '___sec15'),
|
|
('Simple Photon Counts: Bayesian approach in practice',
|
|
3,
|
|
None,
|
|
'___sec16'),
|
|
('Best estimates and confidence intervals', 3, None, '___sec17'),
|
|
('Simple Photon Counts: Best estimates and confidence intervals',
|
|
3,
|
|
None,
|
|
'___sec18'),
|
|
('Bayesian parameter estimation (multiple parameters, '
|
|
'covariance)',
|
|
2,
|
|
None,
|
|
'___sec19'),
|
|
('Bayesian model selection', 2, None, '___sec20')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="Bayesian-bs.html">Data Analysis and Machine Learning: Elements of Bayesian theory and Bayesian Neural Networks</a>
|
|
</div>
|
|
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs001.html#___sec0" style="font-size: 80%;"><b>Why Bayesian Statistics?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs002.html#___sec1" style="font-size: 80%;"><b>Inference</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs003.html#___sec2" style="font-size: 80%;"><b>Statistical Inference</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs004.html#___sec3" style="font-size: 80%;"><b>Some history</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs010.html#___sec4" style="font-size: 80%;"><b>The Bayesian recipe</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs011.html#___sec5" style="font-size: 80%;"><b>Bayes' theorem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs013.html#___sec6" style="font-size: 80%;"><b>The friends of Bayes' theorem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs014.html#___sec7" style="font-size: 80%;"><b>Inference With Parametric Models</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs016.html#___sec8" style="font-size: 80%;"><b>Illustrative examples with python code</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="#___sec9" style="font-size: 80%;"><b>Example: Is this a fair coin?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs018.html#___sec10" style="font-size: 80%;"><b>A few words on different priors</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs019.html#___sec11" style="font-size: 80%;"><b>Bayesian parameter estimation (single parameter)</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs020.html#___sec12" style="font-size: 80%;"> Example: Measured flux from a star</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs020.html#___sec13" style="font-size: 80%;"> Simple Photon Counts: Frequentist Approach</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs020.html#___sec14" style="font-size: 80%;"> Simple Photon Counts: Bayesian Approach</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs020.html#___sec15" style="font-size: 80%;"> A note about priors</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs020.html#___sec16" style="font-size: 80%;"> Simple Photon Counts: Bayesian approach in practice</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs020.html#___sec17" style="font-size: 80%;"> Best estimates and confidence intervals</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs020.html#___sec18" style="font-size: 80%;"> Simple Photon Counts: Best estimates and confidence intervals</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs021.html#___sec19" style="font-size: 80%;"><b>Bayesian parameter estimation (multiple parameters, covariance)</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Bayesian-bs022.html#___sec20" style="font-size: 80%;"><b>Bayesian model selection</b></a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
|
|
<div class="container">
|
|
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
|
|
<a name="part0017"></a>
|
|
<!-- !split -->
|
|
|
|
<h2 id="___sec9" class="anchor">Example: Is this a fair coin? </h2>
|
|
Let us begin with the analysis of data from a simple coin-tossing experiment.
|
|
Given that we had observed 6 heads in 8 flips, would you think it was a fair coin? By fair, we mean that we would be prepared to lay an even 1 : 1 bet on the outcome of a flip being a head or a tail. If we decide that the coin was fair, the question which follows naturally is how sure are we that this was so; if it was not fair, how unfair do we think it was? Furthermore, if we were to continue collecting data for this particular coin, observing the outcomes of additional flips, how would we update our belief on the fairness of the coin?
|
|
|
|
<p>
|
|
A sensible way of formulating this problem is to consider a large number of hypotheses about the range in which the bias-weighting of the coin might lie. If we denote the bias-weighting by \( H \), then \( H = 0 \) and \( H = 1 \) can represent a coin which produces a tail or a head on every flip, respectively. There is a continuum of possibilities for the value of H between these limits, with \( H = 0.5 \) indicating a fair coin. Our state of knowledge about the fairness, or the degree of unfairness, of the coin is then completely summarized by specifying how much we believe these various propositions to be true.
|
|
|
|
<p>
|
|
Let us perform a computer simulation of a coin-tossing experiment. This provides the data that we will be analysing.
|
|
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
|
</pre></div>
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">999</span>) <span style="color: #408080; font-style: italic"># for reproducibility</span>
|
|
a<span style="color: #666666">=0.6</span> <span style="color: #408080; font-style: italic"># biased coin</span>
|
|
flips<span style="color: #666666">=</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">2**12</span>) <span style="color: #408080; font-style: italic"># simulates 4096 coin flips</span>
|
|
heads<span style="color: #666666">=</span>flips<span style="color: #666666"><</span>a <span style="color: #408080; font-style: italic"># boolean array, heads[i]=True if flip i is heads</span>
|
|
</pre></div>
|
|
<p>
|
|
In the light of this data, our inference about the fairness of this coin is summarized by the conditional pdf: \( p(H|D,I) \). This is, of course, shorthand for the limiting case of a continuum of propositions for the value of \( H \); that is to say, the probability that \( H \) lies in an infinitesimally narrow range is given by \( p(H|D,I) dH \).
|
|
|
|
<p>
|
|
To estimate this posterior pdf, we need to use Bayes’ theorem <a href="._Bayesian-bs011.html#mjx-eqn-1">(1)</a>. We will ignore the denominator \( p(D|I) \) as it does not involve bias-weighting explicitly, and it will therefore not affect the shape of the desired pdf. At the end we can evaluate the missing constant subsequently from the normalization condition
|
|
$$
|
|
\begin{equation}
|
|
\int_0^1 p(H|D,I) dH = 1.
|
|
\tag{2}
|
|
\end{equation}
|
|
$$
|
|
|
|
<p>
|
|
The prior pdf, \( p(H|I) \), represents what we know about the coin given only the information \( I \) that we are dealing with a ‘strange coin’. We could keep a very open mind about the nature of the coin; a simple probability assignment which reflects this is a uniform, or flat, prior
|
|
$$
|
|
\begin{equation}
|
|
p(H|I) = \left\{ \begin{array}{ll}
|
|
1 & 0 \le H \le 1, \\
|
|
0 & \mathrm{otherwise}.
|
|
\end{array} \right.
|
|
\tag{3}
|
|
\end{equation}
|
|
$$
|
|
|
|
We will get back later to the choice of prior and its effect on the analysis.
|
|
|
|
<p>
|
|
This prior state of knowledge, or ignorance, is modified by the data through the likelihood function \( p(D|H,I) \). It is a measure of the chance that we would have obtained the data that we actually observed, if the value of the bias-weighting was given (as known). If, in the conditioning information \( I \), we assume that the flips of the coin were independent events, so that the outcome of one did not influence that of another, then the probability of obtaining the data `R heads in N tosses' is given by the binomial distribution (we leave a formal definition of this to a statistics textbook)
|
|
$$
|
|
\begin{equation}
|
|
p(D|H,I) \propto H^R (1-H)^{N-R}.
|
|
\tag{4}
|
|
\end{equation}
|
|
$$
|
|
|
|
It seems reasonable because \( H \) is the chance of obtaining a head on any flip, and there were \( R \) of them, and \( 1-H \) is the corresponding probability for a tail, of which there were \( N-R \). We note that this binomial distribution also contains a normalization factor, but we will ignore it since it does not depend explicitly on \( H \), the quantity of interest. It will be absorbed by the normalization condition <a href="#mjx-eqn-2">(2)</a>.
|
|
|
|
<p>
|
|
We perform the setup of this Bayesian framework on the computer.
|
|
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">prior</span>(H):
|
|
p<span style="color: #666666">=</span>np<span style="color: #666666">.</span>zeros_like(H)
|
|
p[(<span style="color: #666666">0<=</span>x)<span style="color: #666666">&</span>(x<span style="color: #666666"><=1</span>)]<span style="color: #666666">=1</span> <span style="color: #408080; font-style: italic"># allowed range: 0<=H<=1</span>
|
|
<span style="color: #008000; font-weight: bold">return</span> p <span style="color: #408080; font-style: italic"># uniform prior</span>
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">likelihood</span>(H,data):
|
|
N <span style="color: #666666">=</span> <span style="color: #008000">len</span>(data)
|
|
no_of_heads <span style="color: #666666">=</span> <span style="color: #008000">sum</span>(data)
|
|
no_of_tails <span style="color: #666666">=</span> N <span style="color: #666666">-</span> no_of_heads
|
|
<span style="color: #008000; font-weight: bold">return</span> H<span style="color: #666666">**</span>no_of_heads <span style="color: #666666">*</span> (<span style="color: #666666">1-</span>H)<span style="color: #666666">**</span>no_of_tails
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">posterior</span>(H,data):
|
|
p<span style="color: #666666">=</span>prior(H)<span style="color: #666666">*</span>likelihood(H,data)
|
|
norm<span style="color: #666666">=</span>np<span style="color: #666666">.</span>trapz(p,H)
|
|
<span style="color: #008000; font-weight: bold">return</span> p<span style="color: #666666">/</span>norm
|
|
</pre></div>
|
|
<p>
|
|
The next step is to confront this setup with the simulated data. To get a feel for the result, it is instructive to see how the posterior pdf evolves as we obtain more and more data pertaining to the coin. The results of such an analyses is shown in Fig. <a href="#fig:coinflipping">1</a>.
|
|
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>x<span style="color: #666666">=</span>np<span style="color: #666666">.</span>linspace(<span style="color: #666666">0</span>,<span style="color: #666666">1</span>,<span style="color: #666666">100</span>)
|
|
fig, axs <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(nrows<span style="color: #666666">=4</span>,ncols<span style="color: #666666">=3</span>,sharex<span style="color: #666666">=</span><span style="color: #008000">True</span>,sharey<span style="color: #666666">=</span><span style="color: #BA2121">'row'</span>)
|
|
axs_vec<span style="color: #666666">=</span>np<span style="color: #666666">.</span>reshape(axs,<span style="color: #666666">-1</span>)
|
|
axs_vec[<span style="color: #666666">0</span>]<span style="color: #666666">.</span>plot(x,prior(x))
|
|
<span style="color: #008000; font-weight: bold">for</span> ndouble <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">11</span>):
|
|
ax<span style="color: #666666">=</span>axs_vec[<span style="color: #666666">1+</span>ndouble]
|
|
ax<span style="color: #666666">.</span>plot(x,posterior(x,heads[:<span style="color: #666666">2**</span>ndouble]))
|
|
ax<span style="color: #666666">.</span>text(<span style="color: #666666">0.1</span>, <span style="color: #666666">0.8</span>, <span style="color: #BA2121">'$N={0}$'</span><span style="color: #666666">.</span>format(<span style="color: #666666">2**</span>ndouble), transform<span style="color: #666666">=</span>ax<span style="color: #666666">.</span>transAxes)
|
|
<span style="color: #008000; font-weight: bold">for</span> row <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">4</span>): axs[row,<span style="color: #666666">0</span>]<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">'$p(H|D_\mathrm{obs},I)$'</span>)
|
|
<span style="color: #008000; font-weight: bold">for</span> col <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">3</span>): axs[<span style="color: #666666">-1</span>,col]<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">'$H$'</span>)
|
|
</pre></div>
|
|
<p>
|
|
<center> <!-- figure label: --> <div id="fig:coinflipping"></div> <!-- FIGURE -->
|
|
<hr class="figure">
|
|
<center><p class="caption">Figure 1: The evolution of the posterior pdf for the bias-weighting of a coin, as the number of data available increases. The figure on the top left-hand corner of each panel shows the number of data included in the analysis. <!-- caption label: fig:coinflipping --> </p></center>
|
|
<p><img src="fig/coinflipping_fig_1.png" align="bottom" width=500></p>
|
|
</center>
|
|
|
|
<p>
|
|
The panel in the top left-hand corner shows the posterior pdf for \( H \) given no data, i.e., it is the same as the prior pdf of Eq. <a href="#mjx-eqn-3">(3)</a>. It indicates that we have no more reason to believe that the coin is fair than we have to think that it is double-headed, double-tailed, or of any other intermediate bias-weighting.
|
|
|
|
<p>
|
|
The first flip is obviously tails. At this point we have no evidence that the coin has a side with heads, as indicated by the pdf going to zero as \( H \to 1 \). The second flip is obviously heads and we have now excluded both extreme options \( H=0 \) (double-tailed) and \( H=1 \) (double-headed). We can note that the posterior at this point has the simple form \( p(H|D,I) = H(1-H) \) for \( 0 \le H \le 1 \).
|
|
|
|
<p>
|
|
The remainder of Fig. <a href="#fig:coinflipping">1</a> shows how the posterior pdf evolves as the number of data analysed becomes larger and larger. We see that the position of the maximum moves around, but that the amount by which it does so decreases with the increasing number of observations. The width of the posterior pdf also becomes narrower with more data, indicating that we are becoming increasingly confident in our estimate of the bias-weighting. For the coin in this example, the best estimate of \( H \) eventually converges to 0.6, which, of course, was the value chosen to simulate the flips.
|
|
|
|
<p>
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._Bayesian-bs016.html">«</a></li>
|
|
<li><a href="._Bayesian-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._Bayesian-bs009.html">10</a></li>
|
|
<li><a href="._Bayesian-bs010.html">11</a></li>
|
|
<li><a href="._Bayesian-bs011.html">12</a></li>
|
|
<li><a href="._Bayesian-bs012.html">13</a></li>
|
|
<li><a href="._Bayesian-bs013.html">14</a></li>
|
|
<li><a href="._Bayesian-bs014.html">15</a></li>
|
|
<li><a href="._Bayesian-bs015.html">16</a></li>
|
|
<li><a href="._Bayesian-bs016.html">17</a></li>
|
|
<li class="active"><a href="._Bayesian-bs017.html">18</a></li>
|
|
<li><a href="._Bayesian-bs018.html">19</a></li>
|
|
<li><a href="._Bayesian-bs019.html">20</a></li>
|
|
<li><a href="._Bayesian-bs020.html">21</a></li>
|
|
<li><a href="._Bayesian-bs021.html">22</a></li>
|
|
<li><a href="._Bayesian-bs022.html">23</a></li>
|
|
<li><a href="._Bayesian-bs018.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
|
</footer>
|
|
-->
|
|
|
|
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
|
|
|
|
</body>
|
|
</html>
|
|
|
|
|