mhjensen
282 lines
15 KiB
HTML
282 lines
15 KiB
HTML
<!--
|
|
Automatically generated HTML file from DocOnce source
|
|
(https://github.com/hplgit/doconce/)
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
|
|
|
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
|
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
|
|
<style type="text/css">
|
|
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
|
|
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
|
('Thursday September 17', 2, None, '___sec1'),
|
|
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
|
('Various steps in cross-validation', 2, None, '___sec3'),
|
|
('How to set up the cross-validation for Ridge and/or Lasso',
|
|
2,
|
|
None,
|
|
'___sec4'),
|
|
('Cross-validation in brief', 2, None, '___sec5'),
|
|
('Code Example for Cross-validation and $k$-fold '
|
|
'Cross-validation',
|
|
2,
|
|
None,
|
|
'___sec6'),
|
|
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
|
("Another Example from Scikit-Learn's Repository",
|
|
2,
|
|
None,
|
|
'___sec8'),
|
|
('Cross-validation with Ridge', 2, None, '___sec9'),
|
|
('The Ising model', 2, None, '___sec10'),
|
|
('Reformulating the problem to suit regression',
|
|
2,
|
|
None,
|
|
'___sec11'),
|
|
('Linear regression', 2, None, '___sec12'),
|
|
('Singular Value decomposition', 2, None, '___sec13'),
|
|
('The one-dimensional Ising model', 2, None, '___sec14'),
|
|
('Ridge regression', 2, None, '___sec15'),
|
|
('LASSO regression', 2, None, '___sec16'),
|
|
('Performance as function of the regularization parameter',
|
|
2,
|
|
None,
|
|
'___sec17'),
|
|
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
|
('Friday September 18: Intro to Logistic Regression',
|
|
2,
|
|
None,
|
|
'___sec19'),
|
|
('Logistic Regression', 2, None, '___sec20'),
|
|
('Classification problems', 2, None, '___sec21'),
|
|
('Optimization and Deep learning', 2, None, '___sec22'),
|
|
('Basics', 2, None, '___sec23'),
|
|
('Linear classifier', 2, None, '___sec24'),
|
|
('Some selected properties', 2, None, '___sec25'),
|
|
('Simple example', 2, None, '___sec26'),
|
|
('Plotting the mean value for each group', 2, None, '___sec27'),
|
|
('The logistic function', 2, None, '___sec28'),
|
|
('Examples of likelihood functions used in logistic regression '
|
|
'and nueral networks',
|
|
2,
|
|
None,
|
|
'___sec29'),
|
|
('Two parameters', 2, None, '___sec30'),
|
|
('Maximum likelihood', 2, None, '___sec31'),
|
|
('The cost function rewritten', 2, None, '___sec32'),
|
|
('Minimizing the cross entropy', 2, None, '___sec33'),
|
|
('A more compact expression', 2, None, '___sec34'),
|
|
('Extending to more predictors', 2, None, '___sec35'),
|
|
('Including more classes', 2, None, '___sec36'),
|
|
('More classes', 2, None, '___sec37'),
|
|
('Wisconsin Cancer Data', 2, None, '___sec38'),
|
|
('Using the correlation matrix', 2, None, '___sec39'),
|
|
('Discussing the correlation data', 2, None, '___sec40'),
|
|
('Other measures in classification studies: Cancer Data again',
|
|
2,
|
|
None,
|
|
'___sec41')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
|
</div>
|
|
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
|
<!-- navigation toc: --> <li><a href="#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs023.html#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs024.html#___sec23" style="font-size: 80%;">Basics</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs025.html#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs026.html#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs027.html#___sec26" style="font-size: 80%;">Simple example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs028.html#___sec27" style="font-size: 80%;">Plotting the mean value for each group</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs029.html#___sec28" style="font-size: 80%;">The logistic function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs030.html#___sec29" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs031.html#___sec30" style="font-size: 80%;">Two parameters</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs032.html#___sec31" style="font-size: 80%;">Maximum likelihood</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs033.html#___sec32" style="font-size: 80%;">The cost function rewritten</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs034.html#___sec33" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs035.html#___sec34" style="font-size: 80%;">A more compact expression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs036.html#___sec35" style="font-size: 80%;">Extending to more predictors</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs037.html#___sec36" style="font-size: 80%;">Including more classes</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs038.html#___sec37" style="font-size: 80%;">More classes</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs039.html#___sec38" style="font-size: 80%;">Wisconsin Cancer Data</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs040.html#___sec39" style="font-size: 80%;">Using the correlation matrix</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs041.html#___sec40" style="font-size: 80%;">Discussing the correlation data</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs042.html#___sec41" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
|
|
<div class="container">
|
|
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
|
|
<a name="part0005"></a>
|
|
<!-- !split -->
|
|
|
|
<h2 id="___sec4" class="anchor">How to set up the cross-validation for Ridge and/or Lasso </h2>
|
|
|
|
<ul>
|
|
<li> Define a range of interest for the penalty parameter.</li>
|
|
<li> Divide the data set into training and test set comprising samples \( \{1, \ldots, n\} \setminus i \) and \( \{ i \} \), respectively.</li>
|
|
<li> Fit the linear regression model by means of ridge estimation for each \( \lambda \) in the grid using the training set, and the corresponding estimate of the error variance \( \boldsymbol{\sigma}_{-i}^2(\lambda) \), as</li>
|
|
</ul>
|
|
|
|
$$
|
|
\begin{align*}
|
|
\boldsymbol{\beta}_{-i}(\lambda) & = ( \boldsymbol{X}_{-i, \ast}^{T}
|
|
\boldsymbol{X}_{-i, \ast} + \lambda \boldsymbol{I}_{pp})^{-1}
|
|
\boldsymbol{X}_{-i, \ast}^{T} \boldsymbol{y}_{-i}
|
|
\end{align*}
|
|
$$
|
|
|
|
|
|
<ul>
|
|
<li> Evaluate the prediction performance of these models on the test set by \( \log\{L[y_i, \boldsymbol{X}_{i, \ast}; \boldsymbol{\beta}_{-i}(\lambda), \boldsymbol{\sigma}_{-i}^2(\lambda)]\} \). Or, by the prediction error \( |y_i - \boldsymbol{X}_{i, \ast} \boldsymbol{\beta}_{-i}(\lambda)| \), the relative error, the error squared or the R2 score function.</li>
|
|
<li> Repeat the first three steps such that each sample plays the role of the test set once.</li>
|
|
<li> Average the prediction performances of the test sets at each grid point of the penalty bias/parameter. It is an estimate of the prediction performance of the model corresponding to this value of the penalty parameter on novel data. It is defined as</li>
|
|
</ul>
|
|
|
|
$$
|
|
\begin{align*}
|
|
\frac{1}{n} \sum_{i = 1}^n \log\{L[y_i, \mathbf{X}_{i, \ast}; \boldsymbol{\beta}_{-i}(\lambda), \boldsymbol{\sigma}_{-i}^2(\lambda)]\}.
|
|
\end{align*}
|
|
$$
|
|
|
|
<p>
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week38-bs004.html">«</a></li>
|
|
<li><a href="._week38-bs000.html">1</a></li>
|
|
<li><a href="._week38-bs001.html">2</a></li>
|
|
<li><a href="._week38-bs002.html">3</a></li>
|
|
<li><a href="._week38-bs003.html">4</a></li>
|
|
<li><a href="._week38-bs004.html">5</a></li>
|
|
<li class="active"><a href="._week38-bs005.html">6</a></li>
|
|
<li><a href="._week38-bs006.html">7</a></li>
|
|
<li><a href="._week38-bs007.html">8</a></li>
|
|
<li><a href="._week38-bs008.html">9</a></li>
|
|
<li><a href="._week38-bs009.html">10</a></li>
|
|
<li><a href="._week38-bs010.html">11</a></li>
|
|
<li><a href="._week38-bs011.html">12</a></li>
|
|
<li><a href="._week38-bs012.html">13</a></li>
|
|
<li><a href="._week38-bs013.html">14</a></li>
|
|
<li><a href="._week38-bs014.html">15</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week38-bs042.html">43</a></li>
|
|
<li><a href="._week38-bs006.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
|
</footer>
|
|
-->
|
|
|
|
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
|
|
|
|
</body>
|
|
</html>
|
|
|
|
|