mhjensen
295 lines
18 KiB
HTML
295 lines
18 KiB
HTML
<!--
|
|
Automatically generated HTML file from DocOnce source
|
|
(https://github.com/hplgit/doconce/)
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
|
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
|
|
|
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
|
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
|
|
<style type="text/css">
|
|
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
|
|
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
|
('General linear models', 2, None, '___sec1'),
|
|
('Rewriting the fitting procedure as a linear algebra problem',
|
|
2,
|
|
None,
|
|
'___sec2'),
|
|
('Rewriting the fitting procedure as a linear algebra problem, '
|
|
'follows',
|
|
2,
|
|
None,
|
|
'___sec3'),
|
|
('Generalizing the fitting procedure as a linear algebra problem',
|
|
2,
|
|
None,
|
|
'___sec4'),
|
|
('Generalizing the fitting procedure as a linear algebra problem',
|
|
2,
|
|
None,
|
|
'___sec5'),
|
|
('Optimizing our parameters', 2, None, '___sec6'),
|
|
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
|
('Interpretations and optimizing our parameters',
|
|
2,
|
|
None,
|
|
'___sec8'),
|
|
('Interpretations and optimizing our parameters',
|
|
2,
|
|
None,
|
|
'___sec9'),
|
|
('Interpretations and optimizing our parameters',
|
|
2,
|
|
None,
|
|
'___sec10'),
|
|
('The $\\chi^2$ function', 2, None, '___sec11'),
|
|
('The $\\chi^2$ function', 2, None, '___sec12'),
|
|
('The $\\chi^2$ function', 2, None, '___sec13'),
|
|
('The $\\chi^2$ function', 2, None, '___sec14'),
|
|
('The $\\chi^2$ function', 2, None, '___sec15'),
|
|
('The $\\chi^2$ function', 2, None, '___sec16'),
|
|
('Simple regression model', 2, None, '___sec17'),
|
|
('Simple regression model, now using _scikit-learn_',
|
|
2,
|
|
None,
|
|
'___sec18'),
|
|
('Simple linear regression model using _scikit-learn_',
|
|
2,
|
|
None,
|
|
'___sec19'),
|
|
('Simple linear regression model', 2, None, '___sec20'),
|
|
('Less noise', 2, None, '___sec21'),
|
|
('How to study our fits', 2, None, '___sec22'),
|
|
('Minimizing the cost function', 2, None, '___sec23'),
|
|
('Relative error', 2, None, '___sec24'),
|
|
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
|
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
|
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
|
('The mean absolute error and other functions in _scikit-learn_',
|
|
2,
|
|
None,
|
|
'___sec28'),
|
|
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
|
('Simple regression model with gradient descent',
|
|
2,
|
|
None,
|
|
'___sec30'),
|
|
('Simple regression model with stochastic gradient descent',
|
|
2,
|
|
None,
|
|
'___sec31'),
|
|
('Polynomial Regression', 2, None, '___sec32'),
|
|
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
|
('The singular value decompostion', 2, None, '___sec34'),
|
|
('Lasso and Ridge regression', 2, None, '___sec35'),
|
|
('Logistic regression', 2, None, '___sec36')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
|
</div>
|
|
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
|
<!-- navigation toc: --> <li><a href="#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
|
|
<div class="container">
|
|
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
|
|
<a name="part0018"></a>
|
|
<!-- !split -->
|
|
|
|
<h2 id="___sec17" class="anchor">Simple regression model </h2>
|
|
We are now ready to write our first program which aims at solving the above linear regression equations. We start with data we have produced ourselves, in this case normally distributed random numbers along the \( x \)-axis. These numbers define then the value of a function \( y(x)=4+3x+N(0,1) \). Thereafter we order the \( x \) values and employ our linear regression algorithm to set up the best fit. Here we find it useful to use the numpy function \( c\_ \) arrays where arrays are stacked along their last axis after being upgraded to at least two dimensions with ones post-pended to the shape. The following examples help in understanding what happens
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
<span style="color: #008000; font-weight: bold">print</span>(np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>array([<span style="color: #666666">1</span>,<span style="color: #666666">2</span>,<span style="color: #666666">3</span>]), np<span style="color: #666666">.</span>array([<span style="color: #666666">4</span>,<span style="color: #666666">5</span>,<span style="color: #666666">6</span>])])
|
|
<span style="color: #008000; font-weight: bold">print</span>(np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>array([[<span style="color: #666666">1</span>,<span style="color: #666666">2</span>,<span style="color: #666666">3</span>]]), <span style="color: #666666">0</span>, <span style="color: #666666">0</span>, np<span style="color: #666666">.</span>array([[<span style="color: #666666">4</span>,<span style="color: #666666">5</span>,<span style="color: #666666">6</span>]])])
|
|
</pre></div>
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Importing various packages</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> random, seed
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
|
|
|
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
|
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
|
|
|
xb <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)), x]
|
|
beta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(xb<span style="color: #666666">.</span>T<span style="color: #666666">.</span>dot(xb))<span style="color: #666666">.</span>dot(xb<span style="color: #666666">.</span>T)<span style="color: #666666">.</span>dot(y)
|
|
xnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[<span style="color: #666666">0</span>],[<span style="color: #666666">2</span>]])
|
|
xbnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)), xnew]
|
|
ypredict <span style="color: #666666">=</span> xbnew<span style="color: #666666">.</span>dot(beta)
|
|
|
|
plt<span style="color: #666666">.</span>plot(xnew, ypredict, <span style="color: #BA2121">"r-"</span>)
|
|
plt<span style="color: #666666">.</span>plot(x, y ,<span style="color: #BA2121">'ro'</span>)
|
|
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>,<span style="color: #666666">2.0</span>,<span style="color: #666666">0</span>, <span style="color: #666666">15.0</span>])
|
|
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r'$x$'</span>)
|
|
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r'$y$'</span>)
|
|
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">r'Linear Regression'</span>)
|
|
plt<span style="color: #666666">.</span>show()
|
|
</pre></div>
|
|
<p>
|
|
We see that, as expected, a linear fit gives a seemingly (from the graph) good representation of the data.
|
|
|
|
<p>
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._Regression-bs017.html">«</a></li>
|
|
<li><a href="._Regression-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._Regression-bs010.html">11</a></li>
|
|
<li><a href="._Regression-bs011.html">12</a></li>
|
|
<li><a href="._Regression-bs012.html">13</a></li>
|
|
<li><a href="._Regression-bs013.html">14</a></li>
|
|
<li><a href="._Regression-bs014.html">15</a></li>
|
|
<li><a href="._Regression-bs015.html">16</a></li>
|
|
<li><a href="._Regression-bs016.html">17</a></li>
|
|
<li><a href="._Regression-bs017.html">18</a></li>
|
|
<li class="active"><a href="._Regression-bs018.html">19</a></li>
|
|
<li><a href="._Regression-bs019.html">20</a></li>
|
|
<li><a href="._Regression-bs020.html">21</a></li>
|
|
<li><a href="._Regression-bs021.html">22</a></li>
|
|
<li><a href="._Regression-bs022.html">23</a></li>
|
|
<li><a href="._Regression-bs023.html">24</a></li>
|
|
<li><a href="._Regression-bs024.html">25</a></li>
|
|
<li><a href="._Regression-bs025.html">26</a></li>
|
|
<li><a href="._Regression-bs026.html">27</a></li>
|
|
<li><a href="._Regression-bs027.html">28</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._Regression-bs037.html">38</a></li>
|
|
<li><a href="._Regression-bs019.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
|
</footer>
|
|
-->
|
|
|
|
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
|
|
|
|
</body>
|
|
</html>
|
|
|
|
|