Morten Hjorth-Jensen
855 lines
28 KiB
HTML
855 lines
28 KiB
HTML
<!DOCTYPE html>
|
|
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
|
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
|
|
|
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<!-- reveal.js: http://lab.hakim.se/reveal-js/ -->
|
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
|
|
|
|
<meta name="apple-mobile-web-app-capable" content="yes" />
|
|
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, minimal-ui">
|
|
|
|
<link rel="stylesheet" href="reveal.js/css/reveal.css">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/beige.css" id="theme">
|
|
<!--
|
|
<link rel="stylesheet" href="reveal.js/css/reveal.css">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/beige.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/beigesmall.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/solarized.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/serif.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/night.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/moon.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/simple.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/sky.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/darkgray.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/default.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/cbc.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/simula.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/black.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/white.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/league.css" id="theme">
|
|
-->
|
|
|
|
<!-- For syntax highlighting -->
|
|
<link rel="stylesheet" href="reveal.js/lib/css/zenburn.css">
|
|
|
|
<!-- Printing and PDF exports -->
|
|
<script>
|
|
var link = document.createElement( 'link' );
|
|
link.rel = 'stylesheet';
|
|
link.type = 'text/css';
|
|
link.href = window.location.search.match( /print-pdf/gi ) ? 'css/print/pdf.css' : 'css/print/paper.css';
|
|
document.getElementsByTagName( 'head' )[0].appendChild( link );
|
|
</script>
|
|
|
|
<style type="text/css">
|
|
hr { border: 0; width: 80%; border-bottom: 1px solid #aaa}
|
|
p.caption { width: 80%; font-size: 60%; font-style: italic; text-align: left; }
|
|
hr.figure { border: 0; width: 80%; border-bottom: 1px solid #aaa}
|
|
.reveal .alert-text-small { font-size: 80%; }
|
|
.reveal .alert-text-large { font-size: 130%; }
|
|
.reveal .alert-text-normal { font-size: 90%; }
|
|
.reveal .alert {
|
|
padding:8px 35px 8px 14px; margin-bottom:18px;
|
|
text-shadow:0 1px 0 rgba(255,255,255,0.5);
|
|
border:5px solid #bababa;
|
|
-webkit-border-radius: 14px; -moz-border-radius: 14px;
|
|
border-radius:14px;
|
|
background-position: 10px 10px;
|
|
background-repeat: no-repeat;
|
|
background-size: 38px;
|
|
padding-left: 30px; /* 55px; if icon */
|
|
}
|
|
.reveal .alert-block {padding-top:14px; padding-bottom:14px}
|
|
.reveal .alert-block > p, .alert-block > ul {margin-bottom:1em}
|
|
/*.reveal .alert li {margin-top: 1em}*/
|
|
.reveal .alert-block p+p {margin-top:5px}
|
|
/*.reveal .alert-notice { background-image: url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_notice.png); }
|
|
.reveal .alert-summary { background-image:url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_summary.png); }
|
|
.reveal .alert-warning { background-image: url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_warning.png); }
|
|
.reveal .alert-question {background-image:url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_question.png); } */
|
|
|
|
</style>
|
|
|
|
|
|
|
|
<!-- Styles for table layout of slides -->
|
|
<style type="text/css">
|
|
td.padding {
|
|
padding-top:20px;
|
|
padding-bottom:20px;
|
|
padding-right:50px;
|
|
padding-left:50px;
|
|
}
|
|
</style>
|
|
|
|
</head>
|
|
|
|
<body>
|
|
<div class="reveal">
|
|
|
|
<!-- Any section element inside the <div class="slides"> container
|
|
is displayed as a slide -->
|
|
|
|
<div class="slides">
|
|
|
|
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript"
|
|
src="http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<section>
|
|
<!-- ------------------- main content ---------------------- -->
|
|
|
|
|
|
|
|
<center><h1 style="text-align: center;">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</h1></center> <!-- document title -->
|
|
|
|
<p>
|
|
<!-- author(s): Morten Hjorth-Jensen -->
|
|
|
|
<center>
|
|
<b>Morten Hjorth-Jensen</b> [1, 2]
|
|
</center>
|
|
|
|
<p> <br>
|
|
<!-- institution(s) -->
|
|
|
|
<center>[1] <b>Department of Physics, University of Oslo</b></center>
|
|
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
|
<br>
|
|
<p> <br>
|
|
<center><h4>Nov 2, 2017</h4></center> <!-- date -->
|
|
<br>
|
|
<p>
|
|
|
|
<center style="font-size:80%">
|
|
<!-- copyright --> © 1999-2017, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
|
|
</center>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec0">Regression analysis, overarching aims </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
Regression modeling deals with the description of the sampling distribution of a given random variable \( y \) varies as function of another variable or a set of such variables \( \hat{x} =[x_0, x_1,\dots, x_p]^T \).
|
|
The first variable is called the <b>dependent</b>, the <b>outcome</b> or the <b>response</b> variable while the set of variables \( \hat{x} \) is called the independent variable, or the predictor variable or the explanatory variable.
|
|
|
|
<p>
|
|
A regression model aims at finding a likelihood function \( p(y\vert \hat{x}) \), that is the conditional distribution for \( y \) with a given \( \hat{x} \). The estimation of \( p(y\vert \hat{x}) \) is made using a data set with
|
|
|
|
<ul>
|
|
<p><li> \( n \) cases \( i = 0, 1, 2, \dots, n-1 \)</li>
|
|
<p><li> Response (dependent or outcome) variable \( y_i \) with \( i = 0, 1, 2, \dots, n-1 \)</li>
|
|
<p><li> \( p \) Explanatory (independent or predictor) variables \( \hat{x}_i=[x_{i0}, x_{i1}, \dots, x_{ip}] \) with \( i = 0, 1, 2, \dots, n-1 \)</li>
|
|
</ul>
|
|
<p>
|
|
|
|
The goal of the regression analysis is to extract/exploit relationship between \( y_i \) and \( \hat{x}_i \) in or to infer causal dependencies, approximations to the likelihood functions, functional relationships and to make predictions .
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec1">General linear models </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
Before we proceed let us study a case from linear algebra where we aim at fitting a set of data \( \hat{y}=[y_0,y_1,\dots,y_{n-1}] \). We could think of these data as a result of an experiment or a complicated numerical experiment. These data are functions of a series of variables \( \hat{x}=[x_0,x_1,\dots,x_{n-1}] \), that is \( y_i = y(x_i) \) with \( i=0,1,2,\dots,n-1 \). The variables \( x_i \) could represent physical quantities like time, temperature, position etc. We assume that \( y(x) \) is a smooth function.
|
|
|
|
<p>
|
|
Since obtaining these data points may not be trivial, we want to use these data to fit a function which can allow us to make predictions for values of \( y \) which are not in the present set. The perhaps simplest approach is to assume we can parametrize our function in terms of a polynomial of degree \( n-1 \) with \( n \) points, that is
|
|
<p> <br>
|
|
$$
|
|
y=y(x) \rightarrow y(x_i)=\tilde{y}_i+\epsilon_i=\sum_{j=0}^{n-1} \beta_i x_i^j+\epsilon_i,
|
|
$$
|
|
<p> <br>
|
|
|
|
where \( \epsilon_i \) is the error in our approximation.
|
|
|
|
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec2">Rewriting the fitting procedure as a linear algebra problem </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
For every set of values \( y_i,x_i \) we have thus the corresponding set of equations
|
|
<p> <br>
|
|
$$
|
|
\begin{align*}
|
|
y_0&=\beta_0+\beta_1x_0^1+\beta_2x_0^2+\dots+\beta_{n-1}x_0^{n-1}+\epsilon_0\\
|
|
y_1&=\beta_0+\beta_1x_1^1+\beta_2x_1^2+\dots+\beta_{n-1}x_1^{n-1}+\epsilon_1\\
|
|
y_2&=\beta_0+\beta_1x_2^1+\beta_2x_2^2+\dots+\beta_{n-1}x_2^{n-1}+\epsilon_2\\
|
|
\dots & \dots \\
|
|
y_{n-1}&=\beta_0+\beta_1x_{n-1}^1+\beta_2x_{n-1}^2+\dots+\beta_1x_{n-1}^{n-1}+\epsilon_{n-1}.\\
|
|
\end{align*}
|
|
$$
|
|
<p> <br>
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec3">Rewriting the fitting procedure as a linear algebra problem, follows </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
Defining the vectors
|
|
<p> <br>
|
|
$$
|
|
\hat{y} = [y_0,y_1, y_2,\dots, y_{n-1}]^T,
|
|
$$
|
|
<p> <br>
|
|
|
|
<p> <br>
|
|
$$
|
|
\hat{\beta} = [\beta_0,\beta_1, \beta_2,\dots, \beta_{n-1}]^T,
|
|
$$
|
|
<p> <br>
|
|
|
|
<p> <br>
|
|
$$
|
|
\hat{\epsilon} = [\epsilon_0,\epsilon_1, \epsilon_2,\dots, \epsilon_{n-1}]^T,
|
|
$$
|
|
<p> <br>
|
|
|
|
and the matrix
|
|
<p> <br>
|
|
$$
|
|
\hat{X}=
|
|
\begin{bmatrix}
|
|
1& x_{0}^1 &x_{0}^2& \dots & \dots &x_{0}^{n-1}\\
|
|
1& x_{1}^1 &x_{1}^2& \dots & \dots &x_{1}^{n-1}\\
|
|
1& x_{2}^1 &x_{2}^2& \dots & \dots &x_{2}^{n-1}\\
|
|
\dots& \dots &\dots& \dots & \dots &\dots\\
|
|
1& x_{n-1}^1 &x_{n-1}^2& \dots & \dots &x_{n-1}^{n-1}\\
|
|
\end{bmatrix}
|
|
$$
|
|
<p> <br>
|
|
|
|
we can rewrite our equations as
|
|
<p> <br>
|
|
$$
|
|
\hat{y} = \hat{X}\hat{\beta}+\hat{\epsilon}.
|
|
$$
|
|
<p> <br>
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec4">Generalizing the fitting procedure as a linear algebra problem </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
We are obviously not limited to the above polynomial. We could replace the various powers of \( x \) with elements of Fourier series, that is, instead of \( x_i^j \) we could have \( \cos{(j x_i)} \) or \( \sin{(j x_i)} \), or time series or other orthogonal functions.
|
|
For every set of values \( y_i,x_i \) we can then generalize the equations to
|
|
<p> <br>
|
|
$$
|
|
\begin{align*}
|
|
y_0&=\beta_0x_{00}+\beta_1x_{01}+\beta_2x_{02}+\dots+\beta_{n-1}x_{0n-1}+\epsilon_0\\
|
|
y_1&=\beta_0x_{10}+\beta_1x_{11}+\beta_2x_{12}+\dots+\beta_{n-1}x_{1n-1}+\epsilon_1\\
|
|
y_2&=\beta_0x_{20}+\beta_1x_{21}+\beta_2x_{22}+\dots+\beta_{n-1}x_{2n-1}+\epsilon_2\\
|
|
\dots & \dots \\
|
|
y_{i}&=\beta_0x_{i0}+\beta_1x_{i1}+\beta_2x_{i2}+\dots+\beta_{n-1}x_{in-1}+\epsilon_i\\
|
|
\dots & \dots \\
|
|
y_{n-1}&=\beta_0x_{n-1,0}+\beta_1x_{n-1,2}+\beta_2x_{n-1,2}+\dots+\beta_1x_{n-1,n-1}+\epsilon_{n-1}.\\
|
|
\end{align*}
|
|
$$
|
|
<p> <br>
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec5">Generalizing the fitting procedure as a linear algebra problem </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
We redefine in turn the matrix \( \hat{X} \) as
|
|
<p> <br>
|
|
$$
|
|
\hat{X}=
|
|
\begin{bmatrix}
|
|
x_{00}& x_{01} &x_{02}& \dots & \dots &x_{0,n-1}\\
|
|
x_{10}& x_{11} &x_{12}& \dots & \dots &x_{1,n-1}\\
|
|
x_{20}& x_{21} &x_{22}& \dots & \dots &x_{2,n-1}\\
|
|
\dots& \dots &\dots& \dots & \dots &\dots\\
|
|
x_{n-1,0}& x_{n-1,1} &x_{n-1,2}& \dots & \dots &x_{n-1,n-1}\\
|
|
\end{bmatrix}
|
|
$$
|
|
<p> <br>
|
|
|
|
and without loss of generality we rewrite again our equations as
|
|
<p> <br>
|
|
$$
|
|
\hat{y} = \hat{X}\hat{\beta}+\hat{\epsilon}.
|
|
$$
|
|
<p> <br>
|
|
|
|
The left-hand side of this equation forms know. Our error vector \( \hat{\epsilon} \) and the parameter vector \( \hat{\beta} \) are our unknow quantities. How can we obtain the optimal set of \( \beta_i \) values?
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec6">Optimizing our parameters </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
We have defined the matrix \( \hat{X} \)
|
|
<p> <br>
|
|
$$
|
|
\begin{align*}
|
|
y_0&=\beta_0x_{00}+\beta_1x_{01}+\beta_2x_{02}+\dots+\beta_{n-1}x_{0n-1}+\epsilon_0\\
|
|
y_1&=\beta_0x_{10}+\beta_1x_{11}+\beta_2x_{12}+\dots+\beta_{n-1}x_{1n-1}+\epsilon_1\\
|
|
y_2&=\beta_0x_{20}+\beta_1x_{21}+\beta_2x_{22}+\dots+\beta_{n-1}x_{2n-1}+\epsilon_1\\
|
|
\dots & \dots \\
|
|
y_{i}&=\beta_0x_{i0}+\beta_1x_{i1}+\beta_2x_{i2}+\dots+\beta_{n-1}x_{in-1}+\epsilon_1\\
|
|
\dots & \dots \\
|
|
y_{n-1}&=\beta_0x_{n-1,0}+\beta_1x_{n-1,2}+\beta_2x_{n-1,2}+\dots+\beta_1x_{n-1,n-1}+\epsilon_{n-1}.\\
|
|
\end{align*}
|
|
$$
|
|
<p> <br>
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec7">Optimizing our parameters, more details </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
We well use this matrix to define the approximation \( \hat{\tilde{y}} \) via the unknown quantity \( \hat{\beta} \) as
|
|
<p> <br>
|
|
$$
|
|
\hat{\tilde{y}}= \hat{X}\hat{\beta},
|
|
$$
|
|
<p> <br>
|
|
|
|
and in order to find the optimal parameters \( \beta_i \) instead of solving the above linear algebra problem, we define a function which gives a measure of the spread between the values \( y_i \) (which represent hopefully the exact values) and the parametrized values \( \tilde{y}_i \), namely
|
|
<p> <br>
|
|
$$
|
|
Q(\hat{\beta})=\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\left(\hat{y}-\hat{\tilde{y}}\right)^T\left(\hat{y}-\hat{\tilde{y}}\right),
|
|
$$
|
|
<p> <br>
|
|
|
|
or using the matrix \( \hat{X} \) as
|
|
<p> <br>
|
|
$$
|
|
Q(\hat{\beta})=\left(\hat{y}-\hat{X}\hat{\beta}\right)^T\left(\hat{y}-\hat{X}\hat{\beta}\right).
|
|
$$
|
|
<p> <br>
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec8">Interpretations and optimizing our parameters </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
The function
|
|
<p> <br>
|
|
$$
|
|
Q(\hat{\beta})=\left(\hat{y}-\hat{X}\hat{\beta}\right)^T\left(\hat{y}-\hat{X}\hat{\beta}\right),
|
|
$$
|
|
<p> <br>
|
|
|
|
can be linked to the variance of the quantity \( y_i \) if we interpret the latter as the mean value of for example a numerical experiment. When linking below with the maximum likelihood approach below, we will indeed interpret \( y_i \) as a mean value
|
|
<p> <br>
|
|
$$
|
|
y_{i}=\langle y_i \rangle = \beta_0x_{i,0}+\beta_1x_{i,1}+\beta_2x_{i,2}+\dots+\beta_{n-1}x_{i,n-1}+\epsilon_i,
|
|
$$
|
|
<p> <br>
|
|
|
|
where \( \langle y_i \rangle \) is the mean value. Keep in mind also that till now we have treated \( y_i \) as the exact value. Normally, the response (dependent or outcome) variable \( y_i \) the outcome of a numerical experiment or another type of experiment and is thus only an approximation to the true value. It is then always accompanied by an error estimate, often limited to a statistical error estimate given by the standard deviation discussed earlier. In the discussion here we will treat \( y_i \) as our exact value for the response variable.
|
|
|
|
<p>
|
|
In order to find the parameters \( \beta_i \) we will then minimize the spread of \( Q(\hat{\beta}) \) by requiring
|
|
<p> <br>
|
|
$$
|
|
\frac{\partial Q(\hat{\beta})}{\partial \beta_j} = \frac{\partial }{\partial \beta_j}\left[ \sum_{i=0}^{n-1}\left(y_i-\beta_0x_{i,0}-\beta_1x_{i,1}-\beta_2x_{i,2}-\dots-\beta_{n-1}x_{i,n-1}\right)^2\right]=0,
|
|
$$
|
|
<p> <br>
|
|
|
|
which results in
|
|
<p> <br>
|
|
$$
|
|
\frac{\partial Q(\hat{\beta})}{\partial \beta_j} = -2\left[ \sum_{i=0}^{n-1}x_{ij}\left(y_i-\beta_0x_{i,0}-\beta_1x_{i,1}-\beta_2x_{i,2}-\dots-\beta_{n-1}x_{i,n-1}\right)\right]=0,
|
|
$$
|
|
<p> <br>
|
|
|
|
or in a matrix-vector form as
|
|
<p> <br>
|
|
$$
|
|
\frac{\partial Q(\hat{\beta})}{\partial \hat{\beta}} = 0 = \hat{X}^T\left( \hat{y}-\hat{X}\hat{\beta}\right).
|
|
$$
|
|
<p> <br>
|
|
|
|
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec9">Interpretations and optimizing our parameters </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
We can rewrite
|
|
<p> <br>
|
|
$$
|
|
\frac{\partial Q(\hat{\beta})}{\partial \hat{\beta}} = 0 = \hat{X}^T\left( \hat{y}-\hat{X}\hat{\beta}\right),
|
|
$$
|
|
<p> <br>
|
|
|
|
as
|
|
<p> <br>
|
|
$$
|
|
\hat{X}^T\hat{y} = \hat{X}^T\hat{X}\hat{\beta},
|
|
$$
|
|
<p> <br>
|
|
|
|
and if the matrix \( \hat{X}^T\hat{X} \) is invertible we have the solution
|
|
<p> <br>
|
|
$$
|
|
\hat{\beta} =\left(\hat{X}^T\hat{X}\right)^{-1}\hat{X}^T\hat{y}.
|
|
$$
|
|
<p> <br>
|
|
|
|
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec10">Interpretations and optimizing our parameters </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
The residuals \( \hat{\epsilon} \) are in turn given by
|
|
<p> <br>
|
|
$$
|
|
\hat{\epsilon} = \hat{y}-\hat{\tilde{y}} = \hat{y}-\hat{X}\hat{\beta},
|
|
$$
|
|
<p> <br>
|
|
|
|
and with
|
|
<p> <br>
|
|
$$
|
|
\hat{X}^T\left( \hat{y}-\hat{X}\hat{\beta}\right)= 0,
|
|
$$
|
|
<p> <br>
|
|
|
|
we have
|
|
<p> <br>
|
|
$$
|
|
\hat{X}^T\hat{\epsilon}=\hat{X}^T\left( \hat{y}-\hat{X}\hat{\beta}\right)= 0,
|
|
$$
|
|
<p> <br>
|
|
|
|
meaning that the solution for \( \hat{\beta} \) is the one which minimizes the residuals. Later we will link this with the maximum likelihood approach.
|
|
|
|
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec11">The \( \chi^2 \) function </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
Normally, the response (dependent or outcome) variable \( y_i \) the outcome of a numerical experiment or another type of experiment and is thus only an approximation to the true value. It is then always accompanied by an error estimate, often limited to a statistical error estimate given by the standard deviation discussed earlier. In the discussion here we will treat \( y_i \) as our exact value for the response variable.
|
|
|
|
<p>
|
|
Introducing the standard deviation \( \sigma_i \) for each measurement \( y_i \), we define now the \( \chi^2 \) function as
|
|
|
|
<p> <br>
|
|
$$
|
|
\chi^2(\hat{\beta})=\sum_{i=0}^{n-1}\frac{\left(y_i-\tilde{y}_i\right)^2}{\sigma_i^2}=\left(\hat{y}-\hat{\tilde{y}}\right)^T\frac{1}{\hat{\Sigma^2}}\left(\hat{y}-\hat{\tilde{y}}\right),
|
|
$$
|
|
<p> <br>
|
|
|
|
where the matrix \( \hat{\Sigma} \) is a diagonal matrix with \( \sigma_i \) as matrix elements.
|
|
|
|
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec12">The \( \chi^2 \) function </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
In order to find the parameters \( \beta_i \) we will then minimize the spread of \( \chi^2(\hat{\beta}) \) by requiring
|
|
<p> <br>
|
|
$$
|
|
\frac{\partial \chi^2(\hat{\beta})}{\partial \beta_j} = \frac{\partial }{\partial \beta_j}\left[ \sum_{i=0}^{n-1}\left(\frac{y_i-\beta_0x_{i,0}-\beta_1x_{i,1}-\beta_2x_{i,2}-\dots-\beta_{n-1}x_{i,n-1}}{\sigma_i}\right)^2\right]=0,
|
|
$$
|
|
<p> <br>
|
|
|
|
which results in
|
|
<p> <br>
|
|
$$
|
|
\frac{\partial \chi^2(\hat{\beta})}{\partial \beta_j} = -2\left[ \sum_{i=0}^{n-1}\frac{x_{ij}}{\sigma_i}\left(\frac{y_i-\beta_0x_{i,0}-\beta_1x_{i,1}-\beta_2x_{i,2}-\dots-\beta_{n-1}x_{i,n-1}}{\sigma_i}\right)\right]=0,
|
|
$$
|
|
<p> <br>
|
|
|
|
or in a matrix-vector form as
|
|
<p> <br>
|
|
$$
|
|
\frac{\partial \chi^2(\hat{\beta})}{\partial \hat{\beta}} = 0 = \hat{A}^T\left( \hat{b}-\hat{A}\hat{\beta}\right).
|
|
$$
|
|
<p> <br>
|
|
|
|
where we have defined the matrix \( \hat{A} =\hat{X}/\hat{\Sigma} \) with matrix elements \( a_{ij} = x_{ij}/\sigma_i \) and the vector \( \hat{b} \) with elements \( b_i = y_i/\sigma_i \).
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec13">The \( \chi^2 \) function </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
We can rewrite
|
|
<p> <br>
|
|
$$
|
|
\frac{\partial \chi^2(\hat{\beta})}{\partial \hat{\beta}} = 0 = \hat{A}^T\left( \hat{b}-\hat{A}\hat{\beta}\right),
|
|
$$
|
|
<p> <br>
|
|
|
|
as
|
|
<p> <br>
|
|
$$
|
|
\hat{A}^T\hat{b} = \hat{A}^T\hat{A}\hat{\beta},
|
|
$$
|
|
<p> <br>
|
|
|
|
and if the matrix \( \hat{A}^T\hat{A} \) is invertible we have the solution
|
|
<p> <br>
|
|
$$
|
|
\hat{\beta} =\left(\hat{A}^T\hat{A}\right)^{-1}\hat{A}^T\hat{b}.
|
|
$$
|
|
<p> <br>
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec14">The \( \chi^2 \) function </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
If we then introduce the matrix
|
|
<p> <br>
|
|
$$
|
|
\hat{H} = \hat{A}^T\hat{A},
|
|
$$
|
|
<p> <br>
|
|
|
|
we have then the following expression for the parameters \( \beta_j \) (the matrix elements of \( \hat{H} \) are \( h_{ij} \))
|
|
<p> <br>
|
|
$$
|
|
\beta_j = \sum_{k=0}^{p-1}h_{jk}\sum_{i=0}^{n-1}\frac{y_i}{\sigma_i}\frac{x_{ik}}{\sigma_i} = \sum_{k=0}^{p-1}h_{jk}\sum_{i=0}^{n-1}b_ia_{ik}
|
|
$$
|
|
<p> <br>
|
|
|
|
We state without proof the expression for the uncertainty in the parameters \( \beta_j \) as
|
|
<p> <br>
|
|
$$
|
|
\sigma^2(\beta_j) = \sum_{i=0}^{n-1}\sigma_i^2\left( \frac{\partial \beta_j}{\partial y_i}\right)^2,
|
|
$$
|
|
<p> <br>
|
|
|
|
resulting in
|
|
<p> <br>
|
|
$$
|
|
\sigma^2(\beta_j) = \left(\sum_{k=0}^{p-1}h_{jk}\sum_{i=0}^{n-1}a_{ik}\right)\left(\sum_{l=0}^{p-1}h_{jl}\sum_{m=0}^{n-1}a_{ml}\right) = h_{jj}!
|
|
$$
|
|
<p> <br>
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec15">The \( \chi^2 \) function </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
The first step here is to approximate the function \( y \) with a first-order polynomial, that is we write
|
|
<p> <br>
|
|
$$
|
|
y=y(x) \rightarrow y(x_i) \approx \beta_0+\beta_1 x_i.
|
|
$$
|
|
<p> <br>
|
|
|
|
By computing the derivatives of \( \chi^2 \) with respect to \( \beta_0 \) and \( \beta_1 \) show that these are given by
|
|
<p> <br>
|
|
$$
|
|
\frac{\partial \chi^2(\hat{\beta})}{\partial \beta_0} = -2\left[ \sum_{i=0}^{1}\left(\frac{y_i-\beta_0-\beta_1x_{i}}{\sigma_i^2}\right)\right]=0,
|
|
$$
|
|
<p> <br>
|
|
|
|
and
|
|
<p> <br>
|
|
$$
|
|
\frac{\partial \chi^2(\hat{\beta})}{\partial \beta_0} = -2\left[ \sum_{i=0}^{1}x_i\left(\frac{y_i-\beta_0-\beta_1x_{i}}{\sigma_i^2}\right)\right]=0.
|
|
$$
|
|
<p> <br>
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec16">The \( \chi^2 \) function </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
We define then
|
|
<p> <br>
|
|
$$
|
|
\gamma = \sum_{i=0}^{1}\frac{1}{\sigma_i^2},
|
|
$$
|
|
<p> <br>
|
|
|
|
<p> <br>
|
|
$$
|
|
\gamma_x = \sum_{i=0}^{1}\frac{x_{i}}{\sigma_i^2},
|
|
$$
|
|
<p> <br>
|
|
|
|
<p> <br>
|
|
$$
|
|
\gamma_y = \sum_{i=0}^{1}\left(\frac{y_i}{\sigma_i^2}\right),
|
|
$$
|
|
<p> <br>
|
|
|
|
<p> <br>
|
|
$$
|
|
\gamma_{xx} = \sum_{i=0}^{1}\frac{x_ix_{i}}{\sigma_i^2},
|
|
$$
|
|
<p> <br>
|
|
|
|
<p> <br>
|
|
$$
|
|
\gamma_{xy} = \sum_{i=0}^{1}\frac{y_ix_{i}}{\sigma_i^2},
|
|
$$
|
|
<p> <br>
|
|
|
|
and show that
|
|
<p> <br>
|
|
$$
|
|
\beta_0 = \frac{\gamma_{xx}\gamma_y-\gamma_x\gamma_y}{\gamma\gamma_{xx}-\gamma_x^2},
|
|
$$
|
|
<p> <br>
|
|
|
|
<p> <br>
|
|
$$
|
|
\beta_1 = \frac{\gamma_{xy}\gamma-\gamma_x\gamma_y}{\gamma\gamma_{xx}-\gamma_x^2}.
|
|
$$
|
|
<p> <br>
|
|
|
|
<p>
|
|
The LSM suffers often from both being underdetermined and overdetermined in the unknown coefficients \( \beta_i \). A better approach is to use the Singular Value Decomposition (SVD) method discussed below.
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec17">The singular value decompostion </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
How can we use the singular value decomposition to find the parameters \( \beta_j \)? More details will come. We first note that a general \( m\times n \) matrix \( \hat{A} \) can be written in terms of a diagonal matrix \( \hat{\Sigma} \) of dimensionality \( n\times n \) and two orthognal matrices \( \hat{U} \) and \( \hat{V} \), where the first has dimensionality \( m \times n \) and the last dimensionality \( n\times n \). We have then
|
|
<p> <br>
|
|
$$
|
|
\hat{A} = \hat{U}\hat{\Sigma}\hat{V}
|
|
$$
|
|
<p> <br>
|
|
</div>
|
|
</section>
|
|
|
|
|
|
|
|
</div> <!-- class="slides" -->
|
|
</div> <!-- class="reveal" -->
|
|
|
|
<script src="reveal.js/lib/js/head.min.js"></script>
|
|
<script src="reveal.js/js/reveal.js"></script>
|
|
|
|
<script>
|
|
// Full list of configuration options available here:
|
|
// https://github.com/hakimel/reveal.js#configuration
|
|
Reveal.initialize({
|
|
|
|
// Display navigation controls in the bottom right corner
|
|
controls: true,
|
|
|
|
// Display progress bar (below the horiz. slider)
|
|
progress: true,
|
|
|
|
// Display the page number of the current slide
|
|
slideNumber: true,
|
|
|
|
// Push each slide change to the browser history
|
|
history: false,
|
|
|
|
// Enable keyboard shortcuts for navigation
|
|
keyboard: true,
|
|
|
|
// Enable the slide overview mode
|
|
overview: true,
|
|
|
|
// Vertical centering of slides
|
|
//center: true,
|
|
center: false,
|
|
|
|
// Enables touch navigation on devices with touch input
|
|
touch: true,
|
|
|
|
// Loop the presentation
|
|
loop: false,
|
|
|
|
// Change the presentation direction to be RTL
|
|
rtl: false,
|
|
|
|
// Turns fragments on and off globally
|
|
fragments: true,
|
|
|
|
// Flags if the presentation is running in an embedded mode,
|
|
// i.e. contained within a limited portion of the screen
|
|
embedded: false,
|
|
|
|
// Number of milliseconds between automatically proceeding to the
|
|
// next slide, disabled when set to 0, this value can be overwritten
|
|
// by using a data-autoslide attribute on your slides
|
|
autoSlide: 0,
|
|
|
|
// Stop auto-sliding after user input
|
|
autoSlideStoppable: true,
|
|
|
|
// Enable slide navigation via mouse wheel
|
|
mouseWheel: false,
|
|
|
|
// Hides the address bar on mobile devices
|
|
hideAddressBar: true,
|
|
|
|
// Opens links in an iframe preview overlay
|
|
previewLinks: false,
|
|
|
|
// Transition style
|
|
transition: 'default', // default/cube/page/concave/zoom/linear/fade/none
|
|
|
|
// Transition speed
|
|
transitionSpeed: 'default', // default/fast/slow
|
|
|
|
// Transition style for full page slide backgrounds
|
|
backgroundTransition: 'default', // default/none/slide/concave/convex/zoom
|
|
|
|
// Number of slides away from the current that are visible
|
|
viewDistance: 3,
|
|
|
|
// Parallax background image
|
|
//parallaxBackgroundImage: '', // e.g. "'https://s3.amazonaws.com/hakim-static/reveal-js/reveal-parallax-1.jpg'"
|
|
|
|
// Parallax background size
|
|
//parallaxBackgroundSize: '' // CSS syntax, e.g. "2100px 900px"
|
|
|
|
theme: Reveal.getQueryHash().theme, // available themes are in reveal.js/css/theme
|
|
transition: Reveal.getQueryHash().transition || 'default', // default/cube/page/concave/zoom/linear/none
|
|
|
|
});
|
|
|
|
Reveal.initialize({
|
|
dependencies: [
|
|
// Cross-browser shim that fully implements classList - https://github.com/eligrey/classList.js/
|
|
{ src: 'reveal.js/lib/js/classList.js', condition: function() { return !document.body.classList; } },
|
|
|
|
// Interpret Markdown in <section> elements
|
|
{ src: 'reveal.js/plugin/markdown/marked.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
|
|
{ src: 'reveal.js/plugin/markdown/markdown.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
|
|
|
|
// Syntax highlight for <code> elements
|
|
{ src: 'reveal.js/plugin/highlight/highlight.js', async: true, callback: function() { hljs.initHighlightingOnLoad(); } },
|
|
|
|
// Zoom in and out with Alt+click
|
|
{ src: 'reveal.js/plugin/zoom-js/zoom.js', async: true, condition: function() { return !!document.body.classList; } },
|
|
|
|
// Speaker notes
|
|
{ src: 'reveal.js/plugin/notes/notes.js', async: true, condition: function() { return !!document.body.classList; } },
|
|
|
|
// Remote control your reveal.js presentation using a touch device
|
|
//{ src: 'reveal.js/plugin/remotes/remotes.js', async: true, condition: function() { return !!document.body.classList; } },
|
|
|
|
// MathJax
|
|
//{ src: 'reveal.js/plugin/math/math.js', async: true }
|
|
]
|
|
});
|
|
|
|
Reveal.initialize({
|
|
|
|
// The "normal" size of the presentation, aspect ratio will be preserved
|
|
// when the presentation is scaled to fit different resolutions. Can be
|
|
// specified using percentage units.
|
|
width: 1170, // original: 960,
|
|
height: 700,
|
|
|
|
// Factor of the display size that should remain empty around the content
|
|
margin: 0.1,
|
|
|
|
// Bounds for smallest/largest possible scale to apply to content
|
|
minScale: 0.2,
|
|
maxScale: 1.0
|
|
|
|
});
|
|
</script>
|
|
|
|
<!-- begin footer logo
|
|
<div style="position: absolute; bottom: 0px; left: 0; margin-left: 0px">
|
|
<img src="somelogo.png">
|
|
</div>
|
|
end footer logo -->
|
|
|
|
|
|
|
|
</body>
|
|
</html>
|