mhjensen
790 lines
40 KiB
HTML
790 lines
40 KiB
HTML
<!--
|
|
Automatically generated HTML file from DocOnce source
|
|
(https://github.com/hplgit/doconce/)
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
|
|
|
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
|
|
|
|
|
<link href="https://cdn.rawgit.com/hplgit/doconce/master/bundled/html_styles/style_solarized_box/css/solarized_light_code.css" rel="stylesheet" type="text/css" title="light"/>
|
|
<script src="https://cdn.rawgit.com/hplgit/doconce/master/bundled/html_styles/style_solarized_box/js/highlight.pack.js"></script>
|
|
<script>hljs.initHighlightingOnLoad();</script>
|
|
|
|
<link href="https://thomasf.github.io/solarized-css/solarized-light.min.css" rel="stylesheet">
|
|
<style type="text/css">
|
|
h1 {color: #b58900;} /* yellow */
|
|
/* h1 {color: #cb4b16;} orange */
|
|
/* h1 {color: #d33682;} magenta, the original choice of thomasf */
|
|
code { padding: 0px; background-color: inherit; }
|
|
pre {
|
|
border: 0pt solid #93a1a1;
|
|
box-shadow: none;
|
|
}
|
|
|
|
div { text-align: justify; text-justify: inter-word; }
|
|
</style>
|
|
|
|
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
|
('Optimization and Deep learning', 2, None, '___sec1'),
|
|
('Basics', 2, None, '___sec2'),
|
|
('Linear classifier', 2, None, '___sec3'),
|
|
('Some selected properties', 2, None, '___sec4'),
|
|
('The logistic function', 2, None, '___sec5'),
|
|
('Examples of likelihood functions used in logistic regression '
|
|
'and nueral networks',
|
|
2,
|
|
None,
|
|
'___sec6'),
|
|
('Two parameters', 2, None, '___sec7'),
|
|
('Maximum likelihood', 2, None, '___sec8'),
|
|
('The cost function rewritten', 2, None, '___sec9'),
|
|
('Minimizing the cross entropy', 2, None, '___sec10'),
|
|
('A more compact expression', 2, None, '___sec11'),
|
|
('Extending to more predictors', 2, None, '___sec12'),
|
|
('Including more classes', 2, None, '___sec13'),
|
|
('More classes', 2, None, '___sec14'),
|
|
('A simple classification problem', 2, None, '___sec15'),
|
|
('The Credit Card example', 2, None, '___sec16')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "AMS" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
|
|
|
|
<!-- ------------------- main content ---------------------- -->
|
|
|
|
|
|
|
|
<center><h1>Data Analysis and Machine Learning: Logistic Regression</h1></center> <!-- document title -->
|
|
|
|
<p>
|
|
<!-- author(s): Morten Hjorth-Jensen -->
|
|
|
|
<center>
|
|
<b>Morten Hjorth-Jensen</b> [1, 2]
|
|
</center>
|
|
|
|
<p>
|
|
<!-- institution(s) -->
|
|
|
|
<center>[1] <b>Department of Physics, University of Oslo</b></center>
|
|
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
|
<br>
|
|
<p>
|
|
<center><h4>Sep 22, 2019</h4></center> <!-- date -->
|
|
<br>
|
|
<p>
|
|
<!-- !split -->
|
|
|
|
<h2 id="___sec0">Logistic Regression </h2>
|
|
|
|
<p>
|
|
In linear regression our main interest was centered on learning the
|
|
coefficients of a functional fit (say a polynomial) in order to be
|
|
able to predict the response of a continuous variable on some unseen
|
|
data. The fit to the continuous variable \( y_i \) is based on some
|
|
independent variables \( \hat{x}_i \). Linear regression resulted in
|
|
analytical expressions for standard ordinary Least Squares or Ridge
|
|
regression (in terms of matrices to invert) for several quantities,
|
|
ranging from the variance and thereby the confidence intervals of the
|
|
parameters \( \hat{\beta} \) to the mean squared error. If we can invert
|
|
the product of the design matrices, linear regression gives then a
|
|
simple recipe for fitting our data.
|
|
|
|
<p>
|
|
Classification problems, however, are concerned with outcomes taking
|
|
the form of discrete variables (i.e. categories). We may for example,
|
|
on the basis of DNA sequencing for a number of patients, like to find
|
|
out which mutations are important for a certain disease; or based on
|
|
scans of various patients' brains, figure out if there is a tumor or
|
|
not; or given a specific physical system, we'd like to identify its
|
|
state, say whether it is an ordered or disordered system (typical
|
|
situation in solid state physics); or classify the status of a
|
|
patient, whether she/he has a stroke or not and many other similar
|
|
situations.
|
|
|
|
<p>
|
|
The most common situation we encounter when we apply logistic
|
|
regression is that of two possible outcomes, normally denoted as a
|
|
binary outcome, true or false, positive or negative, success or
|
|
failure etc.
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec1">Optimization and Deep learning </h2>
|
|
|
|
<p>
|
|
Logistic regression will also serve as our stepping stone towards
|
|
neural network algorithms and supervised deep learning. For logistic
|
|
learning, the minimization of the cost function leads to a non-linear
|
|
equation in the parameters \( \hat{\beta} \). The optimization of the
|
|
problem calls therefore for minimization algorithms. This forms the
|
|
bottle neck of all machine learning algorithms, namely how to find
|
|
reliable minima of a multi-variable function. This leads us to the
|
|
family of gradient descent methods. The latter are the working horses
|
|
of basically all modern machine learning algorithms.
|
|
|
|
<p>
|
|
We note also that many of the topics discussed here on logistic
|
|
regression are also commonly used in modern supervised Deep Learning
|
|
models, as we will see later.
|
|
|
|
<p>
|
|
<!-- !split -->
|
|
|
|
<h2 id="___sec2">Basics </h2>
|
|
|
|
<p>
|
|
We consider the case where the dependent variables, also called the
|
|
responses or the outcomes, \( y_i \) are discrete and only take values
|
|
from \( k=0,\dots,K-1 \) (i.e. \( K \) classes).
|
|
|
|
<p>
|
|
The goal is to predict the
|
|
output classes from the design matrix \( \hat{X}\in\mathbb{R}^{n\times p} \)
|
|
made of \( n \) samples, each of which carries \( p \) features or predictors. The
|
|
primary goal is to identify the classes to which new unseen samples
|
|
belong.
|
|
|
|
<p>
|
|
Let us specialize to the case of two classes only, with outputs
|
|
\( y_i=0 \) and \( y_i=1 \). Our outcomes could represent the status of a
|
|
credit card user that could default or not on her/his credit card
|
|
debt. That is
|
|
|
|
$$
|
|
y_i = \begin{bmatrix} 0 & \mathrm{no}\\ 1 & \mathrm{yes} \end{bmatrix}.
|
|
$$
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec3">Linear classifier </h2>
|
|
|
|
<p>
|
|
Before moving to the logistic model, let us try to use our linear
|
|
regression model to classify these two outcomes. We could for example
|
|
fit a linear model to the default case if \( y_i > 0.5 \) and the no
|
|
default case \( y_i \leq 0.5 \).
|
|
|
|
<p>
|
|
We would then have our
|
|
weighted linear combination, namely
|
|
$$
|
|
\begin{equation}
|
|
\hat{y} = \hat{X}^T\hat{\beta} + \hat{\epsilon},
|
|
\label{_auto1}
|
|
\end{equation}
|
|
$$
|
|
|
|
where \( \hat{y} \) is a vector representing the possible outcomes, \( \hat{X} \) is our
|
|
\( n\times p \) design matrix and \( \hat{\beta} \) represents our estimators/predictors.
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec4">Some selected properties </h2>
|
|
|
|
<p>
|
|
The main problem with our function is that it takes values on the
|
|
entire real axis. In the case of logistic regression, however, the
|
|
labels \( y_i \) are discrete variables. A typical example is the credit
|
|
card data discussed below here, where we can set the state of
|
|
defaulting the debt to \( y_i=1 \) and not to \( y_i=0 \) for one the persons
|
|
in the data set (see the full example below).
|
|
|
|
<p>
|
|
One simple way to get a discrete output is to have sign
|
|
functions that map the output of a linear regressor to values \( \{0,1\} \),
|
|
\( f(s_i)=sign(s_i)=1 \) if \( s_i\ge 0 \) and 0 if otherwise.
|
|
We will encounter this model in our first demonstration of neural networks. Historically it is called the "perceptron" model in the machine learning
|
|
literature. This model is extremely simple. However, in many cases it is more
|
|
favorable to use a ``soft" classifier that outputs
|
|
the probability of a given category. This leads us to the logistic function.
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec5">The logistic function </h2>
|
|
|
|
<p>
|
|
The perceptron is an example of a ``hard classification" model. We
|
|
will encounter this model when we discuss neural networks as
|
|
well. Each datapoint is deterministically assigned to a category (i.e
|
|
\( y_i=0 \) or \( y_i=1 \)). In many cases, it is favorable to have a "soft"
|
|
classifier that outputs the probability of a given category rather
|
|
than a single value. For example, given \( x_i \), the classifier
|
|
outputs the probability of being in a category \( k \). Logistic regression
|
|
is the most common example of a so-called soft classifier. In logistic
|
|
regression, the probability that a data point \( x_i \)
|
|
belongs to a category \( y_i=\{0,1\} \) is given by the so-called logit function (or Sigmoid) which is meant to represent the likelihood for a given event,
|
|
$$
|
|
p(t) = \frac{1}{1+\mathrm \exp{-t}}=\frac{\exp{t}}{1+\mathrm \exp{t}}.
|
|
$$
|
|
|
|
Note that \( 1-p(t)= p(-t) \).
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec6">Examples of likelihood functions used in logistic regression and nueral networks </h2>
|
|
|
|
<p>
|
|
The following code plots the logistic function, the step function and other functions we will encounter from here and on.
|
|
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #CD5555">"""The sigmoid function (or the logistic curve) is a</span>
|
|
<span style="color: #CD5555">function that takes any real number, z, and outputs a number (0,1).</span>
|
|
<span style="color: #CD5555">It is useful in neural networks for assigning weights on a relative scale.</span>
|
|
<span style="color: #CD5555">The value z is the weighted sum of parameters involved in the learning algorithm."""</span>
|
|
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">math</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">mt</span>
|
|
|
|
z = numpy.arange(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>, .<span style="color: #B452CD">1</span>)
|
|
sigma_fn = numpy.vectorize(<span style="color: #8B008B; font-weight: bold">lambda</span> z: <span style="color: #B452CD">1</span>/(<span style="color: #B452CD">1</span>+numpy.exp(-z)))
|
|
sigma = sigma_fn(z)
|
|
|
|
fig = plt.figure()
|
|
ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
|
|
ax.plot(z, sigma)
|
|
ax.set_ylim([-<span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">1.1</span>])
|
|
ax.set_xlim([-<span style="color: #B452CD">5</span>,<span style="color: #B452CD">5</span>])
|
|
ax.grid(<span style="color: #658b00">True</span>)
|
|
ax.set_xlabel(<span style="color: #CD5555">'z'</span>)
|
|
ax.set_title(<span style="color: #CD5555">'sigmoid function'</span>)
|
|
|
|
plt.show()
|
|
|
|
<span style="color: #CD5555">"""Step Function"""</span>
|
|
z = numpy.arange(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>, .<span style="color: #B452CD">02</span>)
|
|
step_fn = numpy.vectorize(<span style="color: #8B008B; font-weight: bold">lambda</span> z: <span style="color: #B452CD">1.0</span> <span style="color: #8B008B; font-weight: bold">if</span> z >= <span style="color: #B452CD">0.0</span> <span style="color: #8B008B; font-weight: bold">else</span> <span style="color: #B452CD">0.0</span>)
|
|
step = step_fn(z)
|
|
|
|
fig = plt.figure()
|
|
ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
|
|
ax.plot(z, step)
|
|
ax.set_ylim([-<span style="color: #B452CD">0.5</span>, <span style="color: #B452CD">1.5</span>])
|
|
ax.set_xlim([-<span style="color: #B452CD">5</span>,<span style="color: #B452CD">5</span>])
|
|
ax.grid(<span style="color: #658b00">True</span>)
|
|
ax.set_xlabel(<span style="color: #CD5555">'z'</span>)
|
|
ax.set_title(<span style="color: #CD5555">'step function'</span>)
|
|
|
|
plt.show()
|
|
|
|
<span style="color: #CD5555">"""tanh Function"""</span>
|
|
z = numpy.arange(-<span style="color: #B452CD">2</span>*mt.pi, <span style="color: #B452CD">2</span>*mt.pi, <span style="color: #B452CD">0.1</span>)
|
|
t = numpy.tanh(z)
|
|
|
|
fig = plt.figure()
|
|
ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
|
|
ax.plot(z, t)
|
|
ax.set_ylim([-<span style="color: #B452CD">1.0</span>, <span style="color: #B452CD">1.0</span>])
|
|
ax.set_xlim([-<span style="color: #B452CD">2</span>*mt.pi,<span style="color: #B452CD">2</span>*mt.pi])
|
|
ax.grid(<span style="color: #658b00">True</span>)
|
|
ax.set_xlabel(<span style="color: #CD5555">'z'</span>)
|
|
ax.set_title(<span style="color: #CD5555">'tanh function'</span>)
|
|
|
|
plt.show()
|
|
</pre></div>
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec7">Two parameters </h2>
|
|
|
|
<p>
|
|
We assume now that we have two classes with \( y_i \) either \( 0 \) or \( 1 \). Furthermore we assume also that we have only two parameters \( \beta \) in our fitting of the Sigmoid function, that is we define probabilities
|
|
$$
|
|
\begin{align*}
|
|
p(y_i=1|x_i,\hat{\beta}) &= \frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}},\nonumber\\
|
|
p(y_i=0|x_i,\hat{\beta}) &= 1 - p(y_i=1|x_i,\hat{\beta}),
|
|
\end{align*}
|
|
$$
|
|
|
|
where \( \hat{\beta} \) are the weights we wish to extract from data, in our case \( \beta_0 \) and \( \beta_1 \).
|
|
|
|
<p>
|
|
Note that we used
|
|
$$
|
|
p(y_i=0\vert x_i, \hat{\beta}) = 1-p(y_i=1\vert x_i, \hat{\beta}).
|
|
$$
|
|
|
|
<p>
|
|
<!-- !split -->
|
|
|
|
<h2 id="___sec8">Maximum likelihood </h2>
|
|
|
|
<p>
|
|
In order to define the total likelihood for all possible outcomes from a
|
|
dataset \( \mathcal{D}=\{(y_i,x_i)\} \), with the binary labels
|
|
\( y_i\in\{0,1\} \) and where the data points are drawn independently, we use the so-called <a href="https://en.wikipedia.org/wiki/Maximum_likelihood_estimation" target="_blank">Maximum Likelihood Estimation</a> (MLE) principle.
|
|
We aim thus at maximizing
|
|
the probability of seeing the observed data. We can then approximate the
|
|
likelihood in terms of the product of the individual probabilities of a specific outcome \( y_i \), that is
|
|
$$
|
|
\begin{align*}
|
|
P(\mathcal{D}|\hat{\beta})& = \prod_{i=1}^n \left[p(y_i=1|x_i,\hat{\beta})\right]^{y_i}\left[1-p(y_i=1|x_i,\hat{\beta}))\right]^{1-y_i}\nonumber \\
|
|
\end{align*}
|
|
$$
|
|
|
|
from which we obtain the log-likelihood and our <b>cost/loss</b> function
|
|
$$
|
|
\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n \left( y_i\log{p(y_i=1|x_i,\hat{\beta})} + (1-y_i)\log\left[1-p(y_i=1|x_i,\hat{\beta}))\right]\right).
|
|
$$
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec9">The cost function rewritten </h2>
|
|
|
|
<p>
|
|
Reordering the logarithms, we can rewrite the <b>cost/loss</b> function as
|
|
$$
|
|
\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{(1+\exp{(\beta_0+\beta_1x_i)})}\right).
|
|
$$
|
|
|
|
<p>
|
|
The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to \( \beta \).
|
|
Since the cost (error) function is just the negative log-likelihood, for logistic regression we have that
|
|
$$
|
|
\mathcal{C}(\hat{\beta})=-\sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{(1+\exp{(\beta_0+\beta_1x_i)})}\right).
|
|
$$
|
|
|
|
This equation is known in statistics as the <b>cross entropy</b>. Finally, we note that just as in linear regression,
|
|
in practice we often supplement the cross-entropy with additional regularization terms, usually \( L_1 \) and \( L_2 \) regularization as we did for Ridge and Lasso regression.
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec10">Minimizing the cross entropy </h2>
|
|
|
|
<p>
|
|
The cross entropy is a convex function of the weights \( \hat{\beta} \) and,
|
|
therefore, any local minimizer is a global minimizer.
|
|
|
|
<p>
|
|
Minimizing this
|
|
cost function with respect to the two parameters \( \beta_0 \) and \( \beta_1 \) we obtain
|
|
|
|
$$
|
|
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_0} = -\sum_{i=1}^n \left(y_i -\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right),
|
|
$$
|
|
|
|
and
|
|
$$
|
|
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_1} = -\sum_{i=1}^n \left(y_ix_i -x_i\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
|
$$
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec11">A more compact expression </h2>
|
|
|
|
<p>
|
|
Let us now define a vector \( \hat{y} \) with \( n \) elements \( y_i \), an
|
|
\( n\times p \) matrix \( \hat{X} \) which contains the \( x_i \) values and a
|
|
vector \( \hat{p} \) of fitted probabilities \( p(y_i\vert x_i,\hat{\beta}) \). We can rewrite in a more compact form the first
|
|
derivative of cost function as
|
|
|
|
$$
|
|
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}} = -\hat{X}^T\left(\hat{y}-\hat{p}\right).
|
|
$$
|
|
|
|
<p>
|
|
If we in addition define a diagonal matrix \( \hat{W} \) with elements
|
|
\( p(y_i\vert x_i,\hat{\beta})(1-p(y_i\vert x_i,\hat{\beta}) \), we can obtain a compact expression of the second derivative as
|
|
|
|
$$
|
|
\frac{\partial^2 \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}\partial \hat{\beta}^T} = \hat{X}^T\hat{W}\hat{X}.
|
|
$$
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec12">Extending to more predictors </h2>
|
|
|
|
<p>
|
|
Within a binary classification problem, we can easily expand our model to include multiple predictors. Our ratio between likelihoods is then with \( p \) predictors
|
|
$$
|
|
\log{ \frac{p(\hat{\beta}\hat{x})}{1-p(\hat{\beta}\hat{x})}} = \beta_0+\beta_1x_1+\beta_2x_2+\dots+\beta_px_p.
|
|
$$
|
|
|
|
Here we defined \( \hat{x}=[1,x_1,x_2,\dots,x_p] \) and \( \hat{\beta}=[\beta_0, \beta_1, \dots, \beta_p] \) leading to
|
|
$$
|
|
p(\hat{\beta}\hat{x})=\frac{ \exp{(\beta_0+\beta_1x_1+\beta_2x_2+\dots+\beta_px_p)}}{1+\exp{(\beta_0+\beta_1x_1+\beta_2x_2+\dots+\beta_px_p)}}.
|
|
$$
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec13">Including more classes </h2>
|
|
|
|
<p>
|
|
Till now we have mainly focused on two classes, the so-called binary
|
|
system. Suppose we wish to extend to \( K \) classes. Let us for the sake
|
|
of simplicity assume we have only two predictors. We have then
|
|
following model
|
|
|
|
$$
|
|
\log{\frac{p(C=1\vert x)}{p(K\vert x)}} = \beta_{10}+\beta_{11}x_1,
|
|
$$
|
|
|
|
$$
|
|
\log{\frac{p(C=2\vert x)}{p(K\vert x)}} = \beta_{20}+\beta_{21}x_1,
|
|
$$
|
|
|
|
and so on till the class \( C=K-1 \) class
|
|
$$
|
|
\log{\frac{p(C=K-1\vert x)}{p(K\vert x)}} = \beta_{(K-1)0}+\beta_{(K-1)1}x_1,
|
|
$$
|
|
|
|
<p>
|
|
and the model is specified in term of \( K-1 \) so-called log-odds or
|
|
<b>logit</b> transformations.
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec14">More classes </h2>
|
|
|
|
<p>
|
|
In our discussion of neural networks we will encounter the above again
|
|
in terms of a slightly modified function, the so-called <b>Softmax</b> function.
|
|
|
|
<p>
|
|
The softmax function is used in various multiclass classification
|
|
methods, such as multinomial logistic regression (also known as
|
|
softmax regression), multiclass linear discriminant analysis, naive
|
|
Bayes classifiers, and artificial neural networks. Specifically, in
|
|
multinomial logistic regression and linear discriminant analysis, the
|
|
input to the function is the result of \( K \) distinct linear functions,
|
|
and the predicted probability for the \( k \)-th class given a sample
|
|
vector \( \hat{x} \) and a weighting vector \( \hat{\beta} \) is (with two
|
|
predictors):
|
|
|
|
$$
|
|
p(C=k\vert \mathbf {x} )=\frac{\exp{(\beta_{k0}+\beta_{k1}x_1)}}{1+\sum_{l=1}^{K-1}\exp{(\beta_{l0}+\beta_{l1}x_1)}}.
|
|
$$
|
|
|
|
It is easy to extend to more predictors. The final class is
|
|
$$
|
|
p(C=K\vert \mathbf {x} )=\frac{1}{1+\sum_{l=1}^{K-1}\exp{(\beta_{l0}+\beta_{l1}x_1)}},
|
|
$$
|
|
|
|
<p>
|
|
and they sum to one. Our earlier discussions were all specialized to
|
|
the case with two classes only. It is easy to see from the above that
|
|
what we derived earlier is compatible with these equations.
|
|
|
|
<p>
|
|
To find the optimal parameters we would typically use a gradient
|
|
descent method. Newton's method and gradient descent methods are
|
|
discussed in the material on <a href="https://compphysics.github.io/MachineLearning/doc/pub/Splines/html/Splines-bs.html" target="_blank">optimization
|
|
methods</a>.
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec15">A simple classification problem </h2>
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn</span> <span style="color: #8B008B; font-weight: bold">import</span> datasets, linear_model
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">generate_data</span>():
|
|
np.random.seed(<span style="color: #B452CD">0</span>)
|
|
X, y = datasets.make_moons(<span style="color: #B452CD">200</span>, noise=<span style="color: #B452CD">0.20</span>)
|
|
<span style="color: #8B008B; font-weight: bold">return</span> X, y
|
|
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">visualize</span>(X, y, clf):
|
|
plot_decision_boundary(<span style="color: #8B008B; font-weight: bold">lambda</span> x: clf.predict(x), X, y)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">plot_decision_boundary</span>(pred_func, X, y):
|
|
<span style="color: #228B22"># Set min and max values and give it some padding</span>
|
|
x_min, x_max = X[:, <span style="color: #B452CD">0</span>].min() - .<span style="color: #B452CD">5</span>, X[:, <span style="color: #B452CD">0</span>].max() + .<span style="color: #B452CD">5</span>
|
|
y_min, y_max = X[:, <span style="color: #B452CD">1</span>].min() - .<span style="color: #B452CD">5</span>, X[:, <span style="color: #B452CD">1</span>].max() + .<span style="color: #B452CD">5</span>
|
|
h = <span style="color: #B452CD">0.01</span>
|
|
<span style="color: #228B22"># Generate a grid of points with distance h between them</span>
|
|
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
|
|
<span style="color: #228B22"># Predict the function value for the whole gid</span>
|
|
Z = pred_func(np.c_[xx.ravel(), yy.ravel()])
|
|
Z = Z.reshape(xx.shape)
|
|
<span style="color: #228B22"># Plot the contour and training examples</span>
|
|
plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral)
|
|
plt.scatter(X[:, <span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>], c=y, cmap=plt.cm.Spectral)
|
|
plt.show()
|
|
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">classify</span>(X, y):
|
|
clf = linear_model.LogisticRegressionCV()
|
|
clf.fit(X, y)
|
|
<span style="color: #8B008B; font-weight: bold">return</span> clf
|
|
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">main</span>():
|
|
X, y = generate_data()
|
|
<span style="color: #228B22"># visualize(X, y)</span>
|
|
clf = classify(X, y)
|
|
visualize(X, y, clf)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #00688B">__name__</span> == <span style="color: #CD5555">"__main__"</span>:
|
|
main()
|
|
</pre></div>
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec16">The Credit Card example </h2>
|
|
Here we use the the <a href="https://archive.ics.uci.edu/ml/datasets/default+of+credit+card+clients" target="_blank">credit card data</a>. More text to come.
|
|
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
|
|
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> OneHotEncoder
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.compose</span> <span style="color: #8B008B; font-weight: bold">import</span> ColumnTransformer
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> StandardScaler, OneHotEncoder
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> confusion_matrix, accuracy_score, roc_auc_score
|
|
|
|
<span style="color: #228B22"># Trying to set the seed</span>
|
|
np.random.seed(<span style="color: #B452CD">0</span>)
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">random</span>
|
|
random.seed(<span style="color: #B452CD">0</span>)
|
|
|
|
<span style="color: #228B22"># Reading file into data frame</span>
|
|
cwd = os.getcwd()
|
|
filename = cwd + <span style="color: #CD5555">'/default of credit card clients.xls'</span>
|
|
nanDict = {}
|
|
df = pd.read_excel(filename, header=<span style="color: #B452CD">1</span>, skiprows=<span style="color: #B452CD">0</span>, index_col=<span style="color: #B452CD">0</span>, na_values=nanDict)
|
|
|
|
df.rename(index=<span style="color: #658b00">str</span>, columns={<span style="color: #CD5555">"default payment next month"</span>: <span style="color: #CD5555">"defaultPaymentNextMonth"</span>}, inplace=<span style="color: #658b00">True</span>)
|
|
|
|
<span style="color: #228B22"># Features and targets </span>
|
|
X = df.loc[:, df.columns != <span style="color: #CD5555">'defaultPaymentNextMonth'</span>].values
|
|
y = df.loc[:, df.columns == <span style="color: #CD5555">'defaultPaymentNextMonth'</span>].values
|
|
|
|
<span style="color: #228B22"># Categorical variables to one-hot's</span>
|
|
onehotencoder = OneHotEncoder(categories=<span style="color: #CD5555">"auto"</span>)
|
|
|
|
X = ColumnTransformer(
|
|
[(<span style="color: #CD5555">""</span>, onehotencoder, [<span style="color: #B452CD">3</span>]),],
|
|
remainder=<span style="color: #CD5555">"passthrough"</span>
|
|
).fit_transform(X)
|
|
|
|
y.shape
|
|
|
|
<span style="color: #228B22"># Train-test split</span>
|
|
trainingShare = <span style="color: #B452CD">0.5</span>
|
|
seed = <span style="color: #B452CD">1</span>
|
|
XTrain, XTest, yTrain, yTest=train_test_split(X, y, train_size=trainingShare, \
|
|
test_size = <span style="color: #B452CD">1</span>-trainingShare,
|
|
random_state=seed)
|
|
|
|
<span style="color: #228B22"># Input Scaling</span>
|
|
sc = StandardScaler()
|
|
XTrain = sc.fit_transform(XTrain)
|
|
XTest = sc.transform(XTest)
|
|
|
|
<span style="color: #228B22"># One-hot's of the target vector</span>
|
|
Y_train_onehot, Y_test_onehot = onehotencoder.fit_transform(yTrain), onehotencoder.fit_transform(yTest)
|
|
|
|
<span style="color: #228B22"># Remove instances with zeros only for past bill statements or paid amounts</span>
|
|
<span style="color: #CD5555">'''</span>
|
|
<span style="color: #CD5555">df = df.drop(df[(df.BILL_AMT1 == 0) &</span>
|
|
<span style="color: #CD5555"> (df.BILL_AMT2 == 0) &</span>
|
|
<span style="color: #CD5555"> (df.BILL_AMT3 == 0) &</span>
|
|
<span style="color: #CD5555"> (df.BILL_AMT4 == 0) &</span>
|
|
<span style="color: #CD5555"> (df.BILL_AMT5 == 0) &</span>
|
|
<span style="color: #CD5555"> (df.BILL_AMT6 == 0) &</span>
|
|
<span style="color: #CD5555"> (df.PAY_AMT1 == 0) &</span>
|
|
<span style="color: #CD5555"> (df.PAY_AMT2 == 0) &</span>
|
|
<span style="color: #CD5555"> (df.PAY_AMT3 == 0) &</span>
|
|
<span style="color: #CD5555"> (df.PAY_AMT4 == 0) &</span>
|
|
<span style="color: #CD5555"> (df.PAY_AMT5 == 0) &</span>
|
|
<span style="color: #CD5555"> (df.PAY_AMT6 == 0)].index)</span>
|
|
<span style="color: #CD5555">'''</span>
|
|
df = df.drop(df[(df.BILL_AMT1 == <span style="color: #B452CD">0</span>) &
|
|
(df.BILL_AMT2 == <span style="color: #B452CD">0</span>) &
|
|
(df.BILL_AMT3 == <span style="color: #B452CD">0</span>) &
|
|
(df.BILL_AMT4 == <span style="color: #B452CD">0</span>) &
|
|
(df.BILL_AMT5 == <span style="color: #B452CD">0</span>) &
|
|
(df.BILL_AMT6 == <span style="color: #B452CD">0</span>)].index)
|
|
|
|
df = df.drop(df[(df.PAY_AMT1 == <span style="color: #B452CD">0</span>) &
|
|
(df.PAY_AMT2 == <span style="color: #B452CD">0</span>) &
|
|
(df.PAY_AMT3 == <span style="color: #B452CD">0</span>) &
|
|
(df.PAY_AMT4 == <span style="color: #B452CD">0</span>) &
|
|
(df.PAY_AMT5 == <span style="color: #B452CD">0</span>) &
|
|
(df.PAY_AMT6 == <span style="color: #B452CD">0</span>)].index)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> GridSearchCV
|
|
|
|
lambdas=np.logspace(-<span style="color: #B452CD">5</span>,<span style="color: #B452CD">7</span>,<span style="color: #B452CD">13</span>)
|
|
parameters = [{<span style="color: #CD5555">'C'</span>: <span style="color: #B452CD">1.</span>/lambdas, <span style="color: #CD5555">"solver"</span>:[<span style="color: #CD5555">"lbfgs"</span>]}]<span style="color: #228B22">#*len(parameters)}]</span>
|
|
scoring = [<span style="color: #CD5555">'accuracy'</span>, <span style="color: #CD5555">'roc_auc'</span>]
|
|
logReg = LogisticRegression()
|
|
gridSearch = GridSearchCV(logReg, parameters, cv=<span style="color: #B452CD">5</span>, scoring=scoring, refit=<span style="color: #CD5555">'roc_auc'</span>)
|
|
</pre></div>
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #228B22"># "refit" gives the metric used deciding best model. </span>
|
|
<span style="color: #228B22"># See more http://scikit-learn.org/stable/auto_examples/model_selection/plot_multi_metric_evaluation.html</span>
|
|
gridSearch.fit(XTrain, yTrain.ravel())
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">gridSearchSummary</span>(method, scoring):
|
|
<span style="color: #CD5555">"""Prints best parameters from Grid search</span>
|
|
<span style="color: #CD5555"> and AUC with standard deviation for all </span>
|
|
<span style="color: #CD5555"> parameter combos """</span>
|
|
|
|
method = <span style="color: #658b00">eval</span>(method)
|
|
<span style="color: #8B008B; font-weight: bold">if</span> scoring == <span style="color: #CD5555">'accuracy'</span>:
|
|
mean = <span style="color: #CD5555">'mean_test_score'</span>
|
|
sd = <span style="color: #CD5555">'std_test_score'</span>
|
|
<span style="color: #8B008B; font-weight: bold">elif</span> scoring == <span style="color: #CD5555">'auc'</span>:
|
|
mean = <span style="color: #CD5555">'mean_test_roc_auc'</span>
|
|
sd = <span style="color: #CD5555">'std_test_roc_auc'</span>
|
|
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">"Best: %f using %s"</span> % (method.best_score_, method.best_params_))
|
|
means = method.cv_results_[mean]
|
|
stds = method.cv_results_[sd]
|
|
params = method.cv_results_[<span style="color: #CD5555">'params'</span>]
|
|
<span style="color: #8B008B; font-weight: bold">for</span> mean, stdev, param <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(means, stds, params):
|
|
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">"%f (%f) with: %r"</span> % (mean, stdev, param))
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">createConfusionMatrix</span>(method, printOut=<span style="color: #658b00">True</span>):
|
|
<span style="color: #CD5555">"""</span>
|
|
<span style="color: #CD5555"> Computes and prints confusion matrices, accuracy scores,</span>
|
|
<span style="color: #CD5555"> and AUC for test and training sets </span>
|
|
<span style="color: #CD5555"> """</span>
|
|
confusionArray = np.zeros(<span style="color: #B452CD">6</span>, dtype=<span style="color: #658b00">object</span>)
|
|
method = <span style="color: #658b00">eval</span>(method)
|
|
|
|
<span style="color: #228B22"># Train</span>
|
|
yPredTrain = method.predict(XTrain)
|
|
yPredTrain = (yPredTrain > <span style="color: #B452CD">0.5</span>)
|
|
cm = confusion_matrix(
|
|
yTrain, yPredTrain)
|
|
cm = np.around(cm/cm.sum(axis=<span style="color: #B452CD">1</span>)[:,<span style="color: #658b00">None</span>], <span style="color: #B452CD">2</span>)
|
|
confusionArray[<span style="color: #B452CD">0</span>] = cm
|
|
|
|
accScore = accuracy_score(yTrain, yPredTrain)
|
|
confusionArray[<span style="color: #B452CD">1</span>] = accScore
|
|
|
|
AUC = roc_auc_score(yTrain, yPredTrain)
|
|
confusionArray[<span style="color: #B452CD">2</span>] = AUC
|
|
|
|
<span style="color: #8B008B; font-weight: bold">if</span> printOut:
|
|
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">'\n################### Training ###############'</span>)
|
|
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">'\nTraining Confusion matrix: \n'</span>, cm)
|
|
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">'\nTraining Accuracy score: \n'</span>, accScore)
|
|
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">'\nTrain AUC: \n'</span>, AUC)
|
|
|
|
<span style="color: #228B22"># Test</span>
|
|
yPred = method.predict(XTest)
|
|
yPred = (yPred > <span style="color: #B452CD">0.5</span>)
|
|
cm = confusion_matrix(
|
|
yTest, yPred)
|
|
cm = np.around(cm/cm.sum(axis=<span style="color: #B452CD">1</span>)[:,<span style="color: #658b00">None</span>], <span style="color: #B452CD">2</span>)
|
|
confusionArray[<span style="color: #B452CD">3</span>] = cm
|
|
|
|
accScore = accuracy_score(yTest, yPred)
|
|
confusionArray[<span style="color: #B452CD">4</span>] = accScore
|
|
|
|
AUC = roc_auc_score(yTest, yPred)
|
|
confusionArray[<span style="color: #B452CD">5</span>] = AUC
|
|
|
|
<span style="color: #8B008B; font-weight: bold">if</span> printOut:
|
|
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">'\n################### Testing ###############'</span>)
|
|
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">'\nTest Confusion matrix: \n'</span>, cm)
|
|
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">'\nTest Accuracy score: \n'</span>, accScore)
|
|
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">'\nTestAUC: \n'</span>, AUC)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">return</span> confusionArray
|
|
|
|
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">seaborn</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">scikitplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">skplt</span>
|
|
|
|
seaborn.set(style=<span style="color: #CD5555">"white"</span>, context=<span style="color: #CD5555">"notebook"</span>, font_scale=<span style="color: #B452CD">1.5</span>,
|
|
rc={<span style="color: #CD5555">"axes.grid"</span>: <span style="color: #658b00">True</span>, <span style="color: #CD5555">"legend.frameon"</span>: <span style="color: #658b00">False</span>,
|
|
<span style="color: #CD5555">"lines.markeredgewidth"</span>: <span style="color: #B452CD">1.4</span>, <span style="color: #CD5555">"lines.markersize"</span>: <span style="color: #B452CD">10</span>})
|
|
seaborn.set_context(<span style="color: #CD5555">"notebook"</span>, font_scale=<span style="color: #B452CD">1.5</span>, rc={<span style="color: #CD5555">"lines.linewidth"</span>: <span style="color: #B452CD">4.5</span>})
|
|
|
|
yPred = gridSearch.predict_proba(XTest)
|
|
<span style="color: #8B008B; font-weight: bold">print</span>(yTest.ravel().shape, yPred.shape)
|
|
|
|
<span style="color: #228B22">#skplt.metrics.plot_cumulative_gain(yTest.ravel(), yPred_onehot)</span>
|
|
skplt.metrics.plot_cumulative_gain(yTest.ravel(), yPred)
|
|
|
|
defaults = <span style="color: #658b00">sum</span>(yTest == <span style="color: #B452CD">1</span>)
|
|
total = <span style="color: #658b00">len</span>(yTest)
|
|
defaultRate = defaults/total
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">bestCurve</span>(defaults, total, defaultRate):
|
|
x = np.linspace(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>, total)
|
|
|
|
y1 = np.linspace(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>, defaults)
|
|
y2 = np.ones(total-defaults)
|
|
y3 = np.concatenate([y1,y2])
|
|
<span style="color: #8B008B; font-weight: bold">return</span> x, y3
|
|
|
|
x, best = bestCurve(defaults=defaults, total=total, defaultRate=defaultRate)
|
|
plt.plot(x, best)
|
|
|
|
|
|
plt.show()
|
|
</pre></div>
|
|
<p>
|
|
|
|
<!-- ------------------- end of main content --------------- -->
|
|
|
|
|
|
<center style="font-size:80%">
|
|
<!-- copyright --> © 1999-2019, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
|
|
</center>
|
|
|
|
|
|
</body>
|
|
</html>
|
|
|
|
|