added html files
This commit is contained in:
mhjensen
@@ -0,0 +1,285 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0019"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec18" class="anchor">Simple regression model, now using <b>scikit-learn</b> </h2>
|
||||
|
||||
<p>
|
||||
We can repeat the above algorithm using <b>scikit-learn</b> as follows
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Importing various packages</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> random, seed
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LinearRegression
|
||||
|
||||
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
||||
linreg <span style="color: #666666">=</span> LinearRegression()
|
||||
linreg<span style="color: #666666">.</span>fit(x,y)
|
||||
xnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[<span style="color: #666666">0</span>],[<span style="color: #666666">2</span>]])
|
||||
ypredict <span style="color: #666666">=</span> linreg<span style="color: #666666">.</span>predict(xnew)
|
||||
|
||||
plt<span style="color: #666666">.</span>plot(xnew, ypredict, <span style="color: #BA2121">"r-"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x, y ,<span style="color: #BA2121">'ro'</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>,<span style="color: #666666">2.0</span>,<span style="color: #666666">0</span>, <span style="color: #666666">15.0</span>])
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r'$x$'</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r'$y$'</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">r'Random numbers '</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs018.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs011.html">12</a></li>
|
||||
<li><a href="._Regression-bs012.html">13</a></li>
|
||||
<li><a href="._Regression-bs013.html">14</a></li>
|
||||
<li><a href="._Regression-bs014.html">15</a></li>
|
||||
<li><a href="._Regression-bs015.html">16</a></li>
|
||||
<li><a href="._Regression-bs016.html">17</a></li>
|
||||
<li><a href="._Regression-bs017.html">18</a></li>
|
||||
<li><a href="._Regression-bs018.html">19</a></li>
|
||||
<li class="active"><a href="._Regression-bs019.html">20</a></li>
|
||||
<li><a href="._Regression-bs020.html">21</a></li>
|
||||
<li><a href="._Regression-bs021.html">22</a></li>
|
||||
<li><a href="._Regression-bs022.html">23</a></li>
|
||||
<li><a href="._Regression-bs023.html">24</a></li>
|
||||
<li><a href="._Regression-bs024.html">25</a></li>
|
||||
<li><a href="._Regression-bs025.html">26</a></li>
|
||||
<li><a href="._Regression-bs026.html">27</a></li>
|
||||
<li><a href="._Regression-bs027.html">28</a></li>
|
||||
<li><a href="._Regression-bs028.html">29</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs020.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0020"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec19" class="anchor">Simple linear regression model using <b>scikit-learn</b> </h2>
|
||||
|
||||
<p>
|
||||
We start with perhaps our simplest possible example, using <b>scikit-learn</b> to perform linear regression analysis on a data set produced by us.
|
||||
What follows is a simple Python code where we have defined function \( y \) in terms of the variable \( x \). Both are defined as vectors of dimension \( 1\times 100 \). The entries to the vector \( \hat{x} \) are given by random numbers generated with a uniform distribution with entries \( x_i \in [0,1] \) (more about probability distribution functions later). These values are then used to define a function \( y(x) \) (tabulated again as a vector) with a linear dependence on \( x \) plus a random noise added via the normal distribution.
|
||||
|
||||
<p>
|
||||
The Numpy functions are imported used the <b>import numpy as np</b>
|
||||
statement and the random number generator for the uniform distribution
|
||||
is called using the function <b>np.random.rand()</b>, where we specificy
|
||||
that we want \( 100 \) random variables. Using Numpy we define
|
||||
automatically an array with the specified number of elements, \( 100 \) in
|
||||
our case. With the Numpy function <b>randn()</b> we can compute random
|
||||
numbers with the normal distribution (mean value \( \mu \) equal to zero and
|
||||
variance \( \sigma^2 \) set to one) and produce the values of \( y \) assuming a linear
|
||||
dependence as function of \( x \)
|
||||
|
||||
$$
|
||||
y = 2x+N(0,1),
|
||||
$$
|
||||
|
||||
<p>
|
||||
where \( N(0,1) \) represents random numbers generated by the normal
|
||||
distribution. From <b>scikit-learn</b> we import then the
|
||||
<b>LinearRegression</b> functionality and make a prediction \( \tilde{y} =
|
||||
\alpha + \beta x \) using the function <b>fit(x,y)</b>. We call the set of
|
||||
data \( (\hat{x},\hat{y}) \) for our training data. The Python package
|
||||
<b>scikit-learn</b> has also a functionality which extracts the above
|
||||
fitting parameters \( \alpha \) and \( \beta \) (see below). Later we will
|
||||
distinguish between training data and test data.
|
||||
|
||||
<p>
|
||||
For plotting we use the Python package
|
||||
<a href="https://matplotlib.org/" target="_self">matplotlib</a> which produces publication
|
||||
quality figures. Feel free to explore the extensive
|
||||
<a href="https://matplotlib.org/gallery/index.html" target="_self">gallery</a> of examples. In
|
||||
this example we plot our original values of \( x \) and \( y \) as well as the
|
||||
prediction <b>ypredict</b> (\( \tilde{y} \)), which attempts at fitting our
|
||||
data with a straight line.
|
||||
|
||||
<p>
|
||||
The Python code follows here.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Importing various packages</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LinearRegression
|
||||
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> <span style="color: #666666">2*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
||||
linreg <span style="color: #666666">=</span> LinearRegression()
|
||||
linreg<span style="color: #666666">.</span>fit(x,y)
|
||||
xnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[<span style="color: #666666">0</span>],[<span style="color: #666666">1</span>]])
|
||||
ypredict <span style="color: #666666">=</span> linreg<span style="color: #666666">.</span>predict(xnew)
|
||||
|
||||
plt<span style="color: #666666">.</span>plot(xnew, ypredict, <span style="color: #BA2121">"r-"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x, y ,<span style="color: #BA2121">'ro'</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>,<span style="color: #666666">1.0</span>,<span style="color: #666666">0</span>, <span style="color: #666666">5.0</span>])
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r'$x$'</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r'$y$'</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">r'Simple Linear Regression'</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs019.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs012.html">13</a></li>
|
||||
<li><a href="._Regression-bs013.html">14</a></li>
|
||||
<li><a href="._Regression-bs014.html">15</a></li>
|
||||
<li><a href="._Regression-bs015.html">16</a></li>
|
||||
<li><a href="._Regression-bs016.html">17</a></li>
|
||||
<li><a href="._Regression-bs017.html">18</a></li>
|
||||
<li><a href="._Regression-bs018.html">19</a></li>
|
||||
<li><a href="._Regression-bs019.html">20</a></li>
|
||||
<li class="active"><a href="._Regression-bs020.html">21</a></li>
|
||||
<li><a href="._Regression-bs021.html">22</a></li>
|
||||
<li><a href="._Regression-bs022.html">23</a></li>
|
||||
<li><a href="._Regression-bs023.html">24</a></li>
|
||||
<li><a href="._Regression-bs024.html">25</a></li>
|
||||
<li><a href="._Regression-bs025.html">26</a></li>
|
||||
<li><a href="._Regression-bs026.html">27</a></li>
|
||||
<li><a href="._Regression-bs027.html">28</a></li>
|
||||
<li><a href="._Regression-bs028.html">29</a></li>
|
||||
<li><a href="._Regression-bs029.html">30</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs021.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0021"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec20" class="anchor">Simple linear regression model </h2>
|
||||
|
||||
<p>
|
||||
This example serves several aims. It allows us to demonstrate several
|
||||
aspects of data analysis and later machine learning algorithms. The
|
||||
immediate visualization shows that our linear fit is not
|
||||
impressive. It goes through the data points, but there are many
|
||||
outliers which are not reproduced by our linear regression. We could
|
||||
now play around with this small program and change for example the
|
||||
factor in front of \( x \) and the normal distribution. Try to change the
|
||||
function \( y \) to
|
||||
|
||||
$$
|
||||
y = 10x+0.01 \times N(0,1),
|
||||
$$
|
||||
|
||||
<p>
|
||||
where \( x \) is defined as before.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs020.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs013.html">14</a></li>
|
||||
<li><a href="._Regression-bs014.html">15</a></li>
|
||||
<li><a href="._Regression-bs015.html">16</a></li>
|
||||
<li><a href="._Regression-bs016.html">17</a></li>
|
||||
<li><a href="._Regression-bs017.html">18</a></li>
|
||||
<li><a href="._Regression-bs018.html">19</a></li>
|
||||
<li><a href="._Regression-bs019.html">20</a></li>
|
||||
<li><a href="._Regression-bs020.html">21</a></li>
|
||||
<li class="active"><a href="._Regression-bs021.html">22</a></li>
|
||||
<li><a href="._Regression-bs022.html">23</a></li>
|
||||
<li><a href="._Regression-bs023.html">24</a></li>
|
||||
<li><a href="._Regression-bs024.html">25</a></li>
|
||||
<li><a href="._Regression-bs025.html">26</a></li>
|
||||
<li><a href="._Regression-bs026.html">27</a></li>
|
||||
<li><a href="._Regression-bs027.html">28</a></li>
|
||||
<li><a href="._Regression-bs028.html">29</a></li>
|
||||
<li><a href="._Regression-bs029.html">30</a></li>
|
||||
<li><a href="._Regression-bs030.html">31</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs022.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0022"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec21" class="anchor">Less noise </h2>
|
||||
|
||||
<p>
|
||||
Does the fit look better? Indeed, by
|
||||
reducing the role of the normal distribution we see immediately that
|
||||
our linear prediction seemingly reproduces better the training
|
||||
set. However, this testing 'by the eye' is obviouly not satisfactory in the
|
||||
long run. Here we have only defined the training data and our model, and
|
||||
have not discussed a more rigorous approach to the <b>cost</b> function.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs021.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs014.html">15</a></li>
|
||||
<li><a href="._Regression-bs015.html">16</a></li>
|
||||
<li><a href="._Regression-bs016.html">17</a></li>
|
||||
<li><a href="._Regression-bs017.html">18</a></li>
|
||||
<li><a href="._Regression-bs018.html">19</a></li>
|
||||
<li><a href="._Regression-bs019.html">20</a></li>
|
||||
<li><a href="._Regression-bs020.html">21</a></li>
|
||||
<li><a href="._Regression-bs021.html">22</a></li>
|
||||
<li class="active"><a href="._Regression-bs022.html">23</a></li>
|
||||
<li><a href="._Regression-bs023.html">24</a></li>
|
||||
<li><a href="._Regression-bs024.html">25</a></li>
|
||||
<li><a href="._Regression-bs025.html">26</a></li>
|
||||
<li><a href="._Regression-bs026.html">27</a></li>
|
||||
<li><a href="._Regression-bs027.html">28</a></li>
|
||||
<li><a href="._Regression-bs028.html">29</a></li>
|
||||
<li><a href="._Regression-bs029.html">30</a></li>
|
||||
<li><a href="._Regression-bs030.html">31</a></li>
|
||||
<li><a href="._Regression-bs031.html">32</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs023.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0023"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec22" class="anchor">How to study our fits </h2>
|
||||
|
||||
<p>
|
||||
We need more rigorous criteria in defining whether we have succeeded or
|
||||
not in modeling our training data. You will be surprised to see that
|
||||
many scientists seldomly venture beyond this 'by the eye' approach. A
|
||||
standard approach for the <em>cost</em> function is the so-called \( \chi^2 \)
|
||||
function
|
||||
|
||||
$$ \chi^2 = \frac{1}{n}
|
||||
\sum_{i=0}^{n-1}\frac{(y_i-\tilde{y}_i)^2}{\sigma_i^2},
|
||||
$$
|
||||
|
||||
<p>
|
||||
where \( \sigma_i^2 \) is the variance (to be defined later) of the entry
|
||||
\( y_i \). We may not know the explicit value of \( \sigma_i^2 \), it serves
|
||||
however the aim of scaling the equations and make the cost function
|
||||
dimensionless.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs022.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs015.html">16</a></li>
|
||||
<li><a href="._Regression-bs016.html">17</a></li>
|
||||
<li><a href="._Regression-bs017.html">18</a></li>
|
||||
<li><a href="._Regression-bs018.html">19</a></li>
|
||||
<li><a href="._Regression-bs019.html">20</a></li>
|
||||
<li><a href="._Regression-bs020.html">21</a></li>
|
||||
<li><a href="._Regression-bs021.html">22</a></li>
|
||||
<li><a href="._Regression-bs022.html">23</a></li>
|
||||
<li class="active"><a href="._Regression-bs023.html">24</a></li>
|
||||
<li><a href="._Regression-bs024.html">25</a></li>
|
||||
<li><a href="._Regression-bs025.html">26</a></li>
|
||||
<li><a href="._Regression-bs026.html">27</a></li>
|
||||
<li><a href="._Regression-bs027.html">28</a></li>
|
||||
<li><a href="._Regression-bs028.html">29</a></li>
|
||||
<li><a href="._Regression-bs029.html">30</a></li>
|
||||
<li><a href="._Regression-bs030.html">31</a></li>
|
||||
<li><a href="._Regression-bs031.html">32</a></li>
|
||||
<li><a href="._Regression-bs032.html">33</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs024.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0024"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec23" class="anchor">Minimizing the cost function </h2>
|
||||
|
||||
<p>
|
||||
Minimizing the cost function is a central aspect of
|
||||
our discussions to come. Finding its minima as function of the model
|
||||
parameters (\( \alpha \) and \( \beta \) in our case) will be a recurring
|
||||
theme in these series of lectures. Essentially all machine learning
|
||||
algorithms we will discuss center around the minimization of the
|
||||
chosen cost function. This depends in turn on our specific
|
||||
model for describing the data, a typical situation in supervised
|
||||
learning. Automatizing the search for the minima of the cost function is a
|
||||
central ingredient in all algorithms. Typical methods which are
|
||||
employed are various variants of <b>gradient</b> methods. These will be
|
||||
discussed in more detail later. Again, you'll be surprised to hear that
|
||||
many practitioners minimize the above function ''by the eye', popularly dubbed as
|
||||
'chi by the eye'. That is, change a parameter and see (visually and numerically) that
|
||||
the \( \chi^2 \) function becomes smaller.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs023.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs016.html">17</a></li>
|
||||
<li><a href="._Regression-bs017.html">18</a></li>
|
||||
<li><a href="._Regression-bs018.html">19</a></li>
|
||||
<li><a href="._Regression-bs019.html">20</a></li>
|
||||
<li><a href="._Regression-bs020.html">21</a></li>
|
||||
<li><a href="._Regression-bs021.html">22</a></li>
|
||||
<li><a href="._Regression-bs022.html">23</a></li>
|
||||
<li><a href="._Regression-bs023.html">24</a></li>
|
||||
<li class="active"><a href="._Regression-bs024.html">25</a></li>
|
||||
<li><a href="._Regression-bs025.html">26</a></li>
|
||||
<li><a href="._Regression-bs026.html">27</a></li>
|
||||
<li><a href="._Regression-bs027.html">28</a></li>
|
||||
<li><a href="._Regression-bs028.html">29</a></li>
|
||||
<li><a href="._Regression-bs029.html">30</a></li>
|
||||
<li><a href="._Regression-bs030.html">31</a></li>
|
||||
<li><a href="._Regression-bs031.html">32</a></li>
|
||||
<li><a href="._Regression-bs032.html">33</a></li>
|
||||
<li><a href="._Regression-bs033.html">34</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs025.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0025"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec24" class="anchor">Relative error </h2>
|
||||
|
||||
<p>
|
||||
There are many ways to define the cost function. A simpler approach is to look at the relative difference between the training data and the predicted data, that is we define
|
||||
the relative error as
|
||||
|
||||
$$
|
||||
\epsilon_{\mathrm{relative}}= \frac{\vert \hat{y} -\hat{\tilde{y}}\vert}{\vert \hat{y}\vert}.
|
||||
$$
|
||||
|
||||
We can modify easily the above Python code and plot the relative error instead
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LinearRegression
|
||||
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> <span style="color: #666666">5*</span>x<span style="color: #666666">+0.01*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
||||
linreg <span style="color: #666666">=</span> LinearRegression()
|
||||
linreg<span style="color: #666666">.</span>fit(x,y)
|
||||
ypredict <span style="color: #666666">=</span> linreg<span style="color: #666666">.</span>predict(x)
|
||||
|
||||
plt<span style="color: #666666">.</span>plot(x, np<span style="color: #666666">.</span>abs(ypredict<span style="color: #666666">-</span>y)<span style="color: #666666">/</span><span style="color: #008000">abs</span>(y), <span style="color: #BA2121">"ro"</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>,<span style="color: #666666">1.0</span>,<span style="color: #666666">0.0</span>, <span style="color: #666666">0.5</span>])
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r'$x$'</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r'$\epsilon_{\mathrm{relative}}$'</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">r'Relative error'</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
Depending on the parameter in front of the normal distribution, we may
|
||||
have a small or larger relative error. Try to play around with
|
||||
different training data sets and study (graphically) the value of the
|
||||
relative error.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs024.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs017.html">18</a></li>
|
||||
<li><a href="._Regression-bs018.html">19</a></li>
|
||||
<li><a href="._Regression-bs019.html">20</a></li>
|
||||
<li><a href="._Regression-bs020.html">21</a></li>
|
||||
<li><a href="._Regression-bs021.html">22</a></li>
|
||||
<li><a href="._Regression-bs022.html">23</a></li>
|
||||
<li><a href="._Regression-bs023.html">24</a></li>
|
||||
<li><a href="._Regression-bs024.html">25</a></li>
|
||||
<li class="active"><a href="._Regression-bs025.html">26</a></li>
|
||||
<li><a href="._Regression-bs026.html">27</a></li>
|
||||
<li><a href="._Regression-bs027.html">28</a></li>
|
||||
<li><a href="._Regression-bs028.html">29</a></li>
|
||||
<li><a href="._Regression-bs029.html">30</a></li>
|
||||
<li><a href="._Regression-bs030.html">31</a></li>
|
||||
<li><a href="._Regression-bs031.html">32</a></li>
|
||||
<li><a href="._Regression-bs032.html">33</a></li>
|
||||
<li><a href="._Regression-bs033.html">34</a></li>
|
||||
<li><a href="._Regression-bs034.html">35</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs026.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,299 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0026"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec25" class="anchor">The richness of <b>scikit-learn</b> </h2>
|
||||
|
||||
<p>
|
||||
As mentioned above, <b>scikit-learn</b> has an impressive functionality.
|
||||
We can for example extract the values of \( \alpha \) and \( \beta \) and
|
||||
their error estimates, or the variance and standard deviation and many
|
||||
other properties from the statistical data analysis.
|
||||
|
||||
<p>
|
||||
Here we show an
|
||||
example of the functionality of scikit-learn.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LinearRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> mean_squared_error, r2_score, mean_squared_log_error, mean_absolute_error
|
||||
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> <span style="color: #666666">2.0+</span> <span style="color: #666666">5*</span>x<span style="color: #666666">+0.5*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
||||
linreg <span style="color: #666666">=</span> LinearRegression()
|
||||
linreg<span style="color: #666666">.</span>fit(x,y)
|
||||
ypredict <span style="color: #666666">=</span> linreg<span style="color: #666666">.</span>predict(x)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'The intercept alpha: </span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">'</span>, linreg<span style="color: #666666">.</span>intercept_)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'Coefficient beta : </span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">'</span>, linreg<span style="color: #666666">.</span>coef_)
|
||||
<span style="color: #408080; font-style: italic"># The mean squared error </span>
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Mean squared error: </span><span style="color: #BB6688; font-weight: bold">%.2f</span><span style="color: #BA2121">"</span> <span style="color: #666666">%</span> mean_squared_error(y, ypredict))
|
||||
<span style="color: #408080; font-style: italic"># Explained variance score: 1 is perfect prediction </span>
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'Variance score: </span><span style="color: #BB6688; font-weight: bold">%.2f</span><span style="color: #BA2121">'</span> <span style="color: #666666">%</span> r2_score(y, ypredict))
|
||||
<span style="color: #408080; font-style: italic"># Mean squared log error </span>
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'Mean squared log error: </span><span style="color: #BB6688; font-weight: bold">%.2f</span><span style="color: #BA2121">'</span> <span style="color: #666666">%</span> mean_squared_log_error(y, ypredict) )
|
||||
<span style="color: #408080; font-style: italic"># Mean absolute error </span>
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'Mean absolute error: </span><span style="color: #BB6688; font-weight: bold">%.2f</span><span style="color: #BA2121">'</span> <span style="color: #666666">%</span> mean_absolute_error(y, ypredict))
|
||||
plt<span style="color: #666666">.</span>plot(x, ypredict, <span style="color: #BA2121">"r-"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x, y ,<span style="color: #BA2121">'ro'</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0.0</span>,<span style="color: #666666">1.0</span>,<span style="color: #666666">1.5</span>, <span style="color: #666666">7.0</span>])
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r'$x$'</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r'$y$'</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">r'Linear Regression fit '</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs025.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs018.html">19</a></li>
|
||||
<li><a href="._Regression-bs019.html">20</a></li>
|
||||
<li><a href="._Regression-bs020.html">21</a></li>
|
||||
<li><a href="._Regression-bs021.html">22</a></li>
|
||||
<li><a href="._Regression-bs022.html">23</a></li>
|
||||
<li><a href="._Regression-bs023.html">24</a></li>
|
||||
<li><a href="._Regression-bs024.html">25</a></li>
|
||||
<li><a href="._Regression-bs025.html">26</a></li>
|
||||
<li class="active"><a href="._Regression-bs026.html">27</a></li>
|
||||
<li><a href="._Regression-bs027.html">28</a></li>
|
||||
<li><a href="._Regression-bs028.html">29</a></li>
|
||||
<li><a href="._Regression-bs029.html">30</a></li>
|
||||
<li><a href="._Regression-bs030.html">31</a></li>
|
||||
<li><a href="._Regression-bs031.html">32</a></li>
|
||||
<li><a href="._Regression-bs032.html">33</a></li>
|
||||
<li><a href="._Regression-bs033.html">34</a></li>
|
||||
<li><a href="._Regression-bs034.html">35</a></li>
|
||||
<li><a href="._Regression-bs035.html">36</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs027.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0027"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec26" class="anchor">Functions in <b>scikit-learn</b> </h2>
|
||||
|
||||
<p>
|
||||
The function <b>coef</b> gives us the parameter \( \beta \) of our fit while <b>intercept</b> yields
|
||||
\( \alpha \). Depending on the constant in front of the normal distribution, we get values near or far from \( alpha =2 \) and \( \beta =5 \). Try to play around with different parameters in front of the normal distribution. The function <b>meansquarederror</b> gives us the mean square error, a risk metric corresponding to the expected value of the squared (quadratic) error or loss defined as
|
||||
$$ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n}
|
||||
\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2,
|
||||
$$
|
||||
|
||||
<p>
|
||||
The smaller the value, the better the fit. Ideally we would like to
|
||||
have an MSE equal zero. The attentive reader has probably recognized
|
||||
this function as being similar to the \( \chi^2 \) function defined above.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs026.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs019.html">20</a></li>
|
||||
<li><a href="._Regression-bs020.html">21</a></li>
|
||||
<li><a href="._Regression-bs021.html">22</a></li>
|
||||
<li><a href="._Regression-bs022.html">23</a></li>
|
||||
<li><a href="._Regression-bs023.html">24</a></li>
|
||||
<li><a href="._Regression-bs024.html">25</a></li>
|
||||
<li><a href="._Regression-bs025.html">26</a></li>
|
||||
<li><a href="._Regression-bs026.html">27</a></li>
|
||||
<li class="active"><a href="._Regression-bs027.html">28</a></li>
|
||||
<li><a href="._Regression-bs028.html">29</a></li>
|
||||
<li><a href="._Regression-bs029.html">30</a></li>
|
||||
<li><a href="._Regression-bs030.html">31</a></li>
|
||||
<li><a href="._Regression-bs031.html">32</a></li>
|
||||
<li><a href="._Regression-bs032.html">33</a></li>
|
||||
<li><a href="._Regression-bs033.html">34</a></li>
|
||||
<li><a href="._Regression-bs034.html">35</a></li>
|
||||
<li><a href="._Regression-bs035.html">36</a></li>
|
||||
<li><a href="._Regression-bs036.html">37</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs028.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0028"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec27" class="anchor">Other functions in <b>scikit-learn</b> </h2>
|
||||
|
||||
<p>
|
||||
The <b>r2score</b> function computes \( R^2 \), the coefficient of
|
||||
determination. It provides a measure of how well future samples are
|
||||
likely to be predicted by the model. Best possible score is 1.0 and it
|
||||
can be negative (because the model can be arbitrarily worse). A
|
||||
constant model that always predicts the expected value of \( \hat{y} \),
|
||||
disregarding the input features, would get a \( R^2 \) score of \( 0.0 \).
|
||||
|
||||
<p>
|
||||
If \( \tilde{\hat{y}}_i \) is the predicted value of the \( i-th \) sample and \( y_i \) is the corresponding true value, then the score \( R^2 \) is defined as
|
||||
$$
|
||||
R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2},
|
||||
$$
|
||||
|
||||
where we have defined the mean value of \( \hat{y} \) as
|
||||
$$
|
||||
\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i.
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs027.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs020.html">21</a></li>
|
||||
<li><a href="._Regression-bs021.html">22</a></li>
|
||||
<li><a href="._Regression-bs022.html">23</a></li>
|
||||
<li><a href="._Regression-bs023.html">24</a></li>
|
||||
<li><a href="._Regression-bs024.html">25</a></li>
|
||||
<li><a href="._Regression-bs025.html">26</a></li>
|
||||
<li><a href="._Regression-bs026.html">27</a></li>
|
||||
<li><a href="._Regression-bs027.html">28</a></li>
|
||||
<li class="active"><a href="._Regression-bs028.html">29</a></li>
|
||||
<li><a href="._Regression-bs029.html">30</a></li>
|
||||
<li><a href="._Regression-bs030.html">31</a></li>
|
||||
<li><a href="._Regression-bs031.html">32</a></li>
|
||||
<li><a href="._Regression-bs032.html">33</a></li>
|
||||
<li><a href="._Regression-bs033.html">34</a></li>
|
||||
<li><a href="._Regression-bs034.html">35</a></li>
|
||||
<li><a href="._Regression-bs035.html">36</a></li>
|
||||
<li><a href="._Regression-bs036.html">37</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs029.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0029"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec28" class="anchor">The mean absolute error and other functions in <b>scikit-learn</b> </h2>
|
||||
|
||||
<p>
|
||||
Another quantity will meet again in our discussions of regression analysis is
|
||||
mean absolute error (MAE), a risk metric corresponding to the expected value of the absolute error loss or what we call the \( l1 \)-norm loss. In our discussion above we presented the relative error.
|
||||
The MAE is defined as follows
|
||||
$$
|
||||
\text{MAE}(\hat{y}, \hat{\tilde{y}}) = \frac{1}{n} \sum_{i=0}^{n-1} \left| y_i - \tilde{y}_i \right|.
|
||||
$$
|
||||
|
||||
Finally we present the
|
||||
squared logarithmic (quadratic) error
|
||||
$$
|
||||
\text{MSLE}(\hat{y}, \hat{\tilde{y}}) = \frac{1}{n} \sum_{i=0}^{n - 1} (\log_e (1 + y_i) - \log_e (1 + \tilde{y}_i) )^2,
|
||||
$$
|
||||
|
||||
<p>
|
||||
where \( \log_e (x) \) stands for the natural logarithm of \( x \). This error
|
||||
estimate is best to use when targets having exponential growth, such
|
||||
as population counts, average sales of a commodity over a span of
|
||||
years etc.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs028.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs021.html">22</a></li>
|
||||
<li><a href="._Regression-bs022.html">23</a></li>
|
||||
<li><a href="._Regression-bs023.html">24</a></li>
|
||||
<li><a href="._Regression-bs024.html">25</a></li>
|
||||
<li><a href="._Regression-bs025.html">26</a></li>
|
||||
<li><a href="._Regression-bs026.html">27</a></li>
|
||||
<li><a href="._Regression-bs027.html">28</a></li>
|
||||
<li><a href="._Regression-bs028.html">29</a></li>
|
||||
<li class="active"><a href="._Regression-bs029.html">30</a></li>
|
||||
<li><a href="._Regression-bs030.html">31</a></li>
|
||||
<li><a href="._Regression-bs031.html">32</a></li>
|
||||
<li><a href="._Regression-bs032.html">33</a></li>
|
||||
<li><a href="._Regression-bs033.html">34</a></li>
|
||||
<li><a href="._Regression-bs034.html">35</a></li>
|
||||
<li><a href="._Regression-bs035.html">36</a></li>
|
||||
<li><a href="._Regression-bs036.html">37</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs030.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0030"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec29" class="anchor">Cubic polynomial in <b>scikit-learn</b> </h2>
|
||||
|
||||
<p>
|
||||
We will discuss in more
|
||||
detail these and other functions in the various lectures. We conclude this part with another example. Instead of
|
||||
a linear \( x \)-dependence we study now a cubic polynomial and use the polynomial regression analysis tools of scikit-learn.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">random</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> Ridge
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> PolynomialFeatures
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.pipeline</span> <span style="color: #008000; font-weight: bold">import</span> make_pipeline
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LinearRegression
|
||||
|
||||
x<span style="color: #666666">=</span>np<span style="color: #666666">.</span>linspace(<span style="color: #666666">0.02</span>,<span style="color: #666666">0.98</span>,<span style="color: #666666">200</span>)
|
||||
noise <span style="color: #666666">=</span> np<span style="color: #666666">.</span>asarray(random<span style="color: #666666">.</span>sample((<span style="color: #008000">range</span>(<span style="color: #666666">200</span>)),<span style="color: #666666">200</span>))
|
||||
y<span style="color: #666666">=</span>x<span style="color: #666666">**3*</span>noise
|
||||
yn<span style="color: #666666">=</span>x<span style="color: #666666">**3*100</span>
|
||||
poly3 <span style="color: #666666">=</span> PolynomialFeatures(degree<span style="color: #666666">=3</span>)
|
||||
X <span style="color: #666666">=</span> poly3<span style="color: #666666">.</span>fit_transform(x[:,np<span style="color: #666666">.</span>newaxis])
|
||||
clf3 <span style="color: #666666">=</span> LinearRegression()
|
||||
clf3<span style="color: #666666">.</span>fit(X,y)
|
||||
|
||||
Xplot<span style="color: #666666">=</span>poly3<span style="color: #666666">.</span>fit_transform(x[:,np<span style="color: #666666">.</span>newaxis])
|
||||
poly3_plot<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>plot(x, clf3<span style="color: #666666">.</span>predict(Xplot), label<span style="color: #666666">=</span><span style="color: #BA2121">'Cubic Fit'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x,yn, color<span style="color: #666666">=</span><span style="color: #BA2121">'red'</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"True Cubic"</span>)
|
||||
plt<span style="color: #666666">.</span>scatter(x, y, label<span style="color: #666666">=</span><span style="color: #BA2121">'Data'</span>, color<span style="color: #666666">=</span><span style="color: #BA2121">'orange'</span>, s<span style="color: #666666">=15</span>)
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">error</span>(a):
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> y:
|
||||
err<span style="color: #666666">=</span>(y<span style="color: #666666">-</span>yn)<span style="color: #666666">/</span>yn
|
||||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">abs</span>(np<span style="color: #666666">.</span>sum(err))<span style="color: #666666">/</span><span style="color: #008000">len</span>(err)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span> (error(y))
|
||||
</pre></div>
|
||||
<p>
|
||||
Similarly, using <b>R</b>, we can perform similar studies.
|
||||
(more details on <b>R</b> will be inserted later).
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs029.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs022.html">23</a></li>
|
||||
<li><a href="._Regression-bs023.html">24</a></li>
|
||||
<li><a href="._Regression-bs024.html">25</a></li>
|
||||
<li><a href="._Regression-bs025.html">26</a></li>
|
||||
<li><a href="._Regression-bs026.html">27</a></li>
|
||||
<li><a href="._Regression-bs027.html">28</a></li>
|
||||
<li><a href="._Regression-bs028.html">29</a></li>
|
||||
<li><a href="._Regression-bs029.html">30</a></li>
|
||||
<li class="active"><a href="._Regression-bs030.html">31</a></li>
|
||||
<li><a href="._Regression-bs031.html">32</a></li>
|
||||
<li><a href="._Regression-bs032.html">33</a></li>
|
||||
<li><a href="._Regression-bs033.html">34</a></li>
|
||||
<li><a href="._Regression-bs034.html">35</a></li>
|
||||
<li><a href="._Regression-bs035.html">36</a></li>
|
||||
<li><a href="._Regression-bs036.html">37</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs031.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0031"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec30" class="anchor">Simple regression model with gradient descent </h2>
|
||||
Add info about the equations, play around with different learning rates
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Importing various packages</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">math</span> <span style="color: #008000; font-weight: bold">import</span> exp, sqrt
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> random, seed
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
|
||||
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
||||
|
||||
xb <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)), x]
|
||||
theta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(xb<span style="color: #666666">.</span>T<span style="color: #666666">.</span>dot(xb))<span style="color: #666666">.</span>dot(xb<span style="color: #666666">.</span>T)<span style="color: #666666">.</span>dot(y)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(theta_linreg)
|
||||
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
|
||||
|
||||
eta <span style="color: #666666">=</span> <span style="color: #666666">0.1</span>
|
||||
Niterations <span style="color: #666666">=</span> <span style="color: #666666">1000</span>
|
||||
m <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> <span style="color: #008000">iter</span> <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(Niterations):
|
||||
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>m<span style="color: #666666">*</span>xb<span style="color: #666666">.</span>T<span style="color: #666666">.</span>dot(xb<span style="color: #666666">.</span>dot(theta)<span style="color: #666666">-</span>y)
|
||||
theta <span style="color: #666666">-=</span> eta<span style="color: #666666">*</span>gradients
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span>(theta)
|
||||
xnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[<span style="color: #666666">0</span>],[<span style="color: #666666">2</span>]])
|
||||
xbnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)), xnew]
|
||||
ypredict <span style="color: #666666">=</span> xbnew<span style="color: #666666">.</span>dot(theta)
|
||||
ypredict2 <span style="color: #666666">=</span> xbnew<span style="color: #666666">.</span>dot(theta_linreg)
|
||||
plt<span style="color: #666666">.</span>plot(xnew, ypredict, <span style="color: #BA2121">"r-"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(xnew, ypredict2, <span style="color: #BA2121">"b-"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x, y ,<span style="color: #BA2121">'ro'</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>,<span style="color: #666666">2.0</span>,<span style="color: #666666">0</span>, <span style="color: #666666">15.0</span>])
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r'$x$'</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r'$y$'</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">r'Random numbers '</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs030.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs023.html">24</a></li>
|
||||
<li><a href="._Regression-bs024.html">25</a></li>
|
||||
<li><a href="._Regression-bs025.html">26</a></li>
|
||||
<li><a href="._Regression-bs026.html">27</a></li>
|
||||
<li><a href="._Regression-bs027.html">28</a></li>
|
||||
<li><a href="._Regression-bs028.html">29</a></li>
|
||||
<li><a href="._Regression-bs029.html">30</a></li>
|
||||
<li><a href="._Regression-bs030.html">31</a></li>
|
||||
<li class="active"><a href="._Regression-bs031.html">32</a></li>
|
||||
<li><a href="._Regression-bs032.html">33</a></li>
|
||||
<li><a href="._Regression-bs033.html">34</a></li>
|
||||
<li><a href="._Regression-bs034.html">35</a></li>
|
||||
<li><a href="._Regression-bs035.html">36</a></li>
|
||||
<li><a href="._Regression-bs036.html">37</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs032.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0032"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec31" class="anchor">Simple regression model with stochastic gradient descent </h2>
|
||||
Add info about the equations, play around with different learning rates
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Importing various packages</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">math</span> <span style="color: #008000; font-weight: bold">import</span> exp, sqrt
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> random, seed
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> SGDRegressor
|
||||
|
||||
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
||||
|
||||
xb <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)), x]
|
||||
theta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(xb<span style="color: #666666">.</span>T<span style="color: #666666">.</span>dot(xb))<span style="color: #666666">.</span>dot(xb<span style="color: #666666">.</span>T)<span style="color: #666666">.</span>dot(y)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(theta_linreg)
|
||||
sgdreg <span style="color: #666666">=</span> SGDRegressor(n_iter <span style="color: #666666">=</span> <span style="color: #666666">50</span>, penalty<span style="color: #666666">=</span><span style="color: #008000">None</span>, eta0<span style="color: #666666">=0.1</span>)
|
||||
sgdreg<span style="color: #666666">.</span>fit(x,y<span style="color: #666666">.</span>ravel())
|
||||
<span style="color: #008000; font-weight: bold">print</span>(sgdreg<span style="color: #666666">.</span>intercept_, sgdreg<span style="color: #666666">.</span>coef_)
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs031.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs024.html">25</a></li>
|
||||
<li><a href="._Regression-bs025.html">26</a></li>
|
||||
<li><a href="._Regression-bs026.html">27</a></li>
|
||||
<li><a href="._Regression-bs027.html">28</a></li>
|
||||
<li><a href="._Regression-bs028.html">29</a></li>
|
||||
<li><a href="._Regression-bs029.html">30</a></li>
|
||||
<li><a href="._Regression-bs030.html">31</a></li>
|
||||
<li><a href="._Regression-bs031.html">32</a></li>
|
||||
<li class="active"><a href="._Regression-bs032.html">33</a></li>
|
||||
<li><a href="._Regression-bs033.html">34</a></li>
|
||||
<li><a href="._Regression-bs034.html">35</a></li>
|
||||
<li><a href="._Regression-bs035.html">36</a></li>
|
||||
<li><a href="._Regression-bs036.html">37</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs033.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0033"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec32" class="anchor">Polynomial Regression </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Importing various packages</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">math</span> <span style="color: #008000; font-weight: bold">import</span> exp, sqrt
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> random, seed
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
|
||||
m <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(m,<span style="color: #666666">1</span>)<span style="color: #666666">+4.</span>
|
||||
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">*</span>x<span style="color: #666666">+</span> <span style="color: #666666">+</span>x<span style="color: #666666">-</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(m,<span style="color: #666666">1</span>)
|
||||
|
||||
xb <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((m,<span style="color: #666666">1</span>)), x]
|
||||
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(xb<span style="color: #666666">.</span>T<span style="color: #666666">.</span>dot(xb))<span style="color: #666666">.</span>dot(xb<span style="color: #666666">.</span>T)<span style="color: #666666">.</span>dot(y)
|
||||
xnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[<span style="color: #666666">0</span>],[<span style="color: #666666">2</span>]])
|
||||
xbnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)), xnew]
|
||||
ypredict <span style="color: #666666">=</span> xbnew<span style="color: #666666">.</span>dot(theta)
|
||||
|
||||
plt<span style="color: #666666">.</span>plot(xnew, ypredict, <span style="color: #BA2121">"r-"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x, y ,<span style="color: #BA2121">'ro'</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>,<span style="color: #666666">2.0</span>,<span style="color: #666666">0</span>, <span style="color: #666666">15.0</span>])
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r'$x$'</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r'$y$'</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">r'Random numbers '</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs032.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs025.html">26</a></li>
|
||||
<li><a href="._Regression-bs026.html">27</a></li>
|
||||
<li><a href="._Regression-bs027.html">28</a></li>
|
||||
<li><a href="._Regression-bs028.html">29</a></li>
|
||||
<li><a href="._Regression-bs029.html">30</a></li>
|
||||
<li><a href="._Regression-bs030.html">31</a></li>
|
||||
<li><a href="._Regression-bs031.html">32</a></li>
|
||||
<li><a href="._Regression-bs032.html">33</a></li>
|
||||
<li class="active"><a href="._Regression-bs033.html">34</a></li>
|
||||
<li><a href="._Regression-bs034.html">35</a></li>
|
||||
<li><a href="._Regression-bs035.html">36</a></li>
|
||||
<li><a href="._Regression-bs036.html">37</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs034.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0034"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec33" class="anchor">Ridge and Lasso Regression </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> linear_model
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LinearRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> mean_squared_error, r2_score
|
||||
|
||||
<span style="color: #408080; font-style: italic">#creating data with random noise</span>
|
||||
x<span style="color: #666666">=</span>np<span style="color: #666666">.</span>arange(<span style="color: #666666">50</span>)
|
||||
|
||||
delta<span style="color: #666666">=</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>uniform(<span style="color: #666666">-2.5</span>,<span style="color: #666666">2.5</span>, size<span style="color: #666666">=</span>(<span style="color: #666666">50</span>))
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>shuffle(delta)
|
||||
y <span style="color: #666666">=0.5*</span>x<span style="color: #666666">+5+</span>delta
|
||||
|
||||
<span style="color: #408080; font-style: italic">#arranging data into 2x50 matrix</span>
|
||||
a<span style="color: #666666">=</span>np<span style="color: #666666">.</span>array(x) <span style="color: #408080; font-style: italic">#inputs</span>
|
||||
b<span style="color: #666666">=</span>np<span style="color: #666666">.</span>array(y) <span style="color: #408080; font-style: italic">#outputs</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic">#Split into training and test</span>
|
||||
X_train<span style="color: #666666">=</span>a[:<span style="color: #666666">37</span>, np<span style="color: #666666">.</span>newaxis]
|
||||
X_test<span style="color: #666666">=</span>a[<span style="color: #666666">37</span>:, np<span style="color: #666666">.</span>newaxis]
|
||||
y_train<span style="color: #666666">=</span>b[:<span style="color: #666666">37</span>]
|
||||
y_test<span style="color: #666666">=</span>b[<span style="color: #666666">37</span>:]
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"X_train: "</span>, X_train<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"y_train: "</span>, y_train<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"X_test: "</span>, X_test<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"y_test: "</span>, y_test<span style="color: #666666">.</span>shape)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"------------------------------------"</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"Ordinary Least Squares"</span>)
|
||||
<span style="color: #408080; font-style: italic">#Add Ordinary Least Squares fit</span>
|
||||
reg<span style="color: #666666">=</span>LinearRegression()
|
||||
reg<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
pred<span style="color: #666666">=</span>reg<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"Prediction Shape: "</span>, pred<span style="color: #666666">.</span>shape)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'Coefficients: </span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">'</span>, reg<span style="color: #666666">.</span>coef_)
|
||||
<span style="color: #408080; font-style: italic"># The mean squared error</span>
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Mean squared error: </span><span style="color: #BB6688; font-weight: bold">%.2f</span><span style="color: #BA2121">"</span>
|
||||
<span style="color: #666666">%</span> mean_squared_error(y_test, pred))
|
||||
<span style="color: #408080; font-style: italic"># Explained variance score: 1 is perfect prediction</span>
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'Variance score: </span><span style="color: #BB6688; font-weight: bold">%.2f</span><span style="color: #BA2121">'</span> <span style="color: #666666">%</span> r2_score(y_test, pred))
|
||||
|
||||
<span style="color: #408080; font-style: italic">#plot</span>
|
||||
plt<span style="color: #666666">.</span>scatter(X_test,y_test,color<span style="color: #666666">=</span><span style="color: #BA2121">'green'</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Training Data"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(X_test, pred, color<span style="color: #666666">=</span><span style="color: #BA2121">'black'</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Fit Line"</span>)
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"------------------------------------"</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"Ridge Regression"</span>)
|
||||
|
||||
ridge<span style="color: #666666">=</span>linear_model<span style="color: #666666">.</span>RidgeCV(alphas<span style="color: #666666">=</span>[<span style="color: #666666">0.1</span>,<span style="color: #666666">1.0</span>,<span style="color: #666666">10.0</span>])
|
||||
ridge<span style="color: #666666">.</span>fit(X_train,y_train)
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"Ridge Coefficient: "</span>,ridge<span style="color: #666666">.</span>coef_)
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"Ridge Intercept: "</span>, ridge<span style="color: #666666">.</span>intercept_)
|
||||
<span style="color: #408080; font-style: italic">#Look into graphing with Ridge fit</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"------------------------------------"</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"Lasso"</span>)
|
||||
lasso<span style="color: #666666">=</span>linear_model<span style="color: #666666">.</span>Lasso(alpha<span style="color: #666666">=0.1</span>)
|
||||
lasso<span style="color: #666666">.</span>fit(X_train,y_train)
|
||||
predl<span style="color: #666666">=</span>lasso<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Lasso Coefficient: "</span>, lasso<span style="color: #666666">.</span>coef_)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Lasso Intercept: "</span>, lasso<span style="color: #666666">.</span>intercept_)
|
||||
plt<span style="color: #666666">.</span>scatter(X_test,y_test,color<span style="color: #666666">=</span><span style="color: #BA2121">'green'</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Training Data"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(X_test, predl, color<span style="color: #666666">=</span><span style="color: #BA2121">'blue'</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Lasso"</span>)
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs033.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs026.html">27</a></li>
|
||||
<li><a href="._Regression-bs027.html">28</a></li>
|
||||
<li><a href="._Regression-bs028.html">29</a></li>
|
||||
<li><a href="._Regression-bs029.html">30</a></li>
|
||||
<li><a href="._Regression-bs030.html">31</a></li>
|
||||
<li><a href="._Regression-bs031.html">32</a></li>
|
||||
<li><a href="._Regression-bs032.html">33</a></li>
|
||||
<li><a href="._Regression-bs033.html">34</a></li>
|
||||
<li class="active"><a href="._Regression-bs034.html">35</a></li>
|
||||
<li><a href="._Regression-bs035.html">36</a></li>
|
||||
<li><a href="._Regression-bs036.html">37</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs035.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,263 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0035"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec34" class="anchor">The singular value decompostion </h2>
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-body">
|
||||
<p> <!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
|
||||
How can we use the singular value decomposition to find the parameters \( \beta_j \)? More details will come. We first note that a general \( m\times n \) matrix \( \hat{A} \) can be written in terms of a diagonal matrix \( \hat{\Sigma} \) of dimensionality \( n\times n \) and two orthognal matrices \( \hat{U} \) and \( \hat{V} \), where the first has dimensionality \( m \times n \) and the last dimensionality \( n\times n \). We have then
|
||||
$$
|
||||
\hat{A} = \hat{U}\hat{\Sigma}\hat{V}
|
||||
$$
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<p>
|
||||
Add codes and discuss this in connection with lasso and ridge, show example where the standard inversion of a matrix fails and where SVD comes to rescue
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs034.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs027.html">28</a></li>
|
||||
<li><a href="._Regression-bs028.html">29</a></li>
|
||||
<li><a href="._Regression-bs029.html">30</a></li>
|
||||
<li><a href="._Regression-bs030.html">31</a></li>
|
||||
<li><a href="._Regression-bs031.html">32</a></li>
|
||||
<li><a href="._Regression-bs032.html">33</a></li>
|
||||
<li><a href="._Regression-bs033.html">34</a></li>
|
||||
<li><a href="._Regression-bs034.html">35</a></li>
|
||||
<li class="active"><a href="._Regression-bs035.html">36</a></li>
|
||||
<li><a href="._Regression-bs036.html">37</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs036.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs037.html#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0036"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec35" class="anchor">Lasso and Ridge regression </h2>
|
||||
|
||||
<p>
|
||||
Discuss the mathematics here
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs035.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs028.html">29</a></li>
|
||||
<li><a href="._Regression-bs029.html">30</a></li>
|
||||
<li><a href="._Regression-bs030.html">31</a></li>
|
||||
<li><a href="._Regression-bs031.html">32</a></li>
|
||||
<li><a href="._Regression-bs032.html">33</a></li>
|
||||
<li><a href="._Regression-bs033.html">34</a></li>
|
||||
<li><a href="._Regression-bs034.html">35</a></li>
|
||||
<li><a href="._Regression-bs035.html">36</a></li>
|
||||
<li class="active"><a href="._Regression-bs036.html">37</a></li>
|
||||
<li><a href="._Regression-bs037.html">38</a></li>
|
||||
<li><a href="._Regression-bs037.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Regression analysis, overarching aims', 2, None, '___sec0'),
|
||||
('General linear models', 2, None, '___sec1'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec2'),
|
||||
('Rewriting the fitting procedure as a linear algebra problem, '
|
||||
'follows',
|
||||
2,
|
||||
None,
|
||||
'___sec3'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Generalizing the fitting procedure as a linear algebra problem',
|
||||
2,
|
||||
None,
|
||||
'___sec5'),
|
||||
('Optimizing our parameters', 2, None, '___sec6'),
|
||||
('Optimizing our parameters, more details', 2, None, '___sec7'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec9'),
|
||||
('Interpretations and optimizing our parameters',
|
||||
2,
|
||||
None,
|
||||
'___sec10'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec11'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec12'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec13'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec14'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec15'),
|
||||
('The $\\chi^2$ function', 2, None, '___sec16'),
|
||||
('Simple regression model', 2, None, '___sec17'),
|
||||
('Simple regression model, now using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec18'),
|
||||
('Simple linear regression model using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Simple linear regression model', 2, None, '___sec20'),
|
||||
('Less noise', 2, None, '___sec21'),
|
||||
('How to study our fits', 2, None, '___sec22'),
|
||||
('Minimizing the cost function', 2, None, '___sec23'),
|
||||
('Relative error', 2, None, '___sec24'),
|
||||
('The richness of _scikit-learn_', 2, None, '___sec25'),
|
||||
('Functions in _scikit-learn_', 2, None, '___sec26'),
|
||||
('Other functions in _scikit-learn_', 2, None, '___sec27'),
|
||||
('The mean absolute error and other functions in _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'___sec28'),
|
||||
('Cubic polynomial in _scikit-learn_', 2, None, '___sec29'),
|
||||
('Simple regression model with gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Simple regression model with stochastic gradient descent',
|
||||
2,
|
||||
None,
|
||||
'___sec31'),
|
||||
('Polynomial Regression', 2, None, '___sec32'),
|
||||
('Ridge and Lasso Regression', 2, None, '___sec33'),
|
||||
('The singular value decompostion', 2, None, '___sec34'),
|
||||
('Lasso and Ridge regression', 2, None, '___sec35'),
|
||||
('Logistic regression', 2, None, '___sec36')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="Regression-bs.html">Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs001.html#___sec0" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs002.html#___sec1" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs003.html#___sec2" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs004.html#___sec3" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, follows</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs005.html#___sec4" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs006.html#___sec5" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs007.html#___sec6" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs008.html#___sec7" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs009.html#___sec8" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs010.html#___sec9" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs011.html#___sec10" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs012.html#___sec11" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs013.html#___sec12" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs014.html#___sec13" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs015.html#___sec14" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs016.html#___sec15" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs017.html#___sec16" style="font-size: 80%;">The \( \chi^2 \) function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs018.html#___sec17" style="font-size: 80%;">Simple regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs019.html#___sec18" style="font-size: 80%;">Simple regression model, now using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs020.html#___sec19" style="font-size: 80%;">Simple linear regression model using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs021.html#___sec20" style="font-size: 80%;">Simple linear regression model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs022.html#___sec21" style="font-size: 80%;">Less noise</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs023.html#___sec22" style="font-size: 80%;">How to study our fits</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs024.html#___sec23" style="font-size: 80%;">Minimizing the cost function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs025.html#___sec24" style="font-size: 80%;">Relative error</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs026.html#___sec25" style="font-size: 80%;">The richness of <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs027.html#___sec26" style="font-size: 80%;">Functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs028.html#___sec27" style="font-size: 80%;">Other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs029.html#___sec28" style="font-size: 80%;">The mean absolute error and other functions in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs030.html#___sec29" style="font-size: 80%;">Cubic polynomial in <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs031.html#___sec30" style="font-size: 80%;">Simple regression model with gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs032.html#___sec31" style="font-size: 80%;">Simple regression model with stochastic gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs033.html#___sec32" style="font-size: 80%;">Polynomial Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs034.html#___sec33" style="font-size: 80%;">Ridge and Lasso Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs035.html#___sec34" style="font-size: 80%;">The singular value decompostion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Regression-bs036.html#___sec35" style="font-size: 80%;">Lasso and Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec36" style="font-size: 80%;">Logistic regression</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0037"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec36" class="anchor">Logistic regression </h2>
|
||||
Add discussion about classification versus regression, show examples of more than two cases and why regression is not the best approach. Motivate for k-nearest neighbors
|
||||
|
||||
<p>
|
||||
Add examples on classification problems
|
||||
|
||||
<p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._Regression-bs036.html">«</a></li>
|
||||
<li><a href="._Regression-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._Regression-bs029.html">30</a></li>
|
||||
<li><a href="._Regression-bs030.html">31</a></li>
|
||||
<li><a href="._Regression-bs031.html">32</a></li>
|
||||
<li><a href="._Regression-bs032.html">33</a></li>
|
||||
<li><a href="._Regression-bs033.html">34</a></li>
|
||||
<li><a href="._Regression-bs034.html">35</a></li>
|
||||
<li><a href="._Regression-bs035.html">36</a></li>
|
||||
<li><a href="._Regression-bs036.html">37</a></li>
|
||||
<li class="active"><a href="._Regression-bs037.html">38</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -393,7 +393,7 @@ and
|
||||
We define then
|
||||
!bt
|
||||
\[
|
||||
\gamma = \sum_{i=0}^{1}\frac{n-1}{\sigma_i^2},
|
||||
\gamma = \sum_{i=0}^{n-1}\frac{n-1}{\sigma_i^2},
|
||||
\]
|
||||
!et
|
||||
|
||||
|
||||
Reference in New Issue
Block a user