added dot files
This commit is contained in:
mhjensen
@@ -0,0 +1,272 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
||||
('Thursday September 17', 2, None, '___sec1'),
|
||||
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
||||
('Various steps in cross-validation', 2, None, '___sec3'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Cross-validation in brief', 2, None, '___sec5'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Cross-validation with Ridge', 2, None, '___sec9'),
|
||||
('The Ising model', 2, None, '___sec10'),
|
||||
('Reformulating the problem to suit regression',
|
||||
2,
|
||||
None,
|
||||
'___sec11'),
|
||||
('Linear regression', 2, None, '___sec12'),
|
||||
('Singular Value decomposition', 2, None, '___sec13'),
|
||||
('The one-dimensional Ising model', 2, None, '___sec14'),
|
||||
('Ridge regression', 2, None, '___sec15'),
|
||||
('LASSO regression', 2, None, '___sec16'),
|
||||
('Performance as function of the regularization parameter',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
||||
('Friday September 18: Intro to Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Logistic Regression', 2, None, '___sec20'),
|
||||
('Classification problems', 2, None, '___sec21'),
|
||||
('Optimization and Deep learning', 2, None, '___sec22'),
|
||||
('Basics', 2, None, '___sec23'),
|
||||
('Linear classifier', 2, None, '___sec24'),
|
||||
('Some selected properties', 2, None, '___sec25'),
|
||||
('The logistic function', 2, None, '___sec26'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'___sec27'),
|
||||
('Two parameters', 2, None, '___sec28'),
|
||||
('Maximum likelihood', 2, None, '___sec29'),
|
||||
('The cost function rewritten', 2, None, '___sec30'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec31'),
|
||||
('A more compact expression', 2, None, '___sec32'),
|
||||
('Extending to more predictors', 2, None, '___sec33'),
|
||||
('Including more classes', 2, None, '___sec34'),
|
||||
('More classes', 2, None, '___sec35'),
|
||||
('A simple classification problem', 2, None, '___sec36'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec37'),
|
||||
('Other measures in classification studies: Cancer Data again',
|
||||
2,
|
||||
None,
|
||||
'___sec38')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs005.html#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs024.html#___sec23" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs025.html#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs026.html#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs027.html#___sec26" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs028.html#___sec27" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs029.html#___sec28" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs030.html#___sec29" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs031.html#___sec30" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs032.html#___sec31" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs033.html#___sec32" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs034.html#___sec33" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs035.html#___sec34" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs036.html#___sec35" style="font-size: 80%;">More classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs037.html#___sec36" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs038.html#___sec37" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs039.html#___sec38" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0023"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec22" class="anchor">Optimization and Deep learning </h2>
|
||||
|
||||
<p>
|
||||
Logistic regression will also serve as our stepping stone towards
|
||||
neural network algorithms and supervised deep learning. For logistic
|
||||
learning, the minimization of the cost function leads to a non-linear
|
||||
equation in the parameters \( \hat{\beta} \). The optimization of the
|
||||
problem calls therefore for minimization algorithms. This forms the
|
||||
bottle neck of all machine learning algorithms, namely how to find
|
||||
reliable minima of a multi-variable function. This leads us to the
|
||||
family of gradient descent methods. The latter are the working horses
|
||||
of basically all modern machine learning algorithms.
|
||||
|
||||
<p>
|
||||
We note also that many of the topics discussed here on logistic
|
||||
regression are also commonly used in modern supervised Deep Learning
|
||||
models, as we will see later.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week38-bs022.html">«</a></li>
|
||||
<li><a href="._week38-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs015.html">16</a></li>
|
||||
<li><a href="._week38-bs016.html">17</a></li>
|
||||
<li><a href="._week38-bs017.html">18</a></li>
|
||||
<li><a href="._week38-bs018.html">19</a></li>
|
||||
<li><a href="._week38-bs019.html">20</a></li>
|
||||
<li><a href="._week38-bs020.html">21</a></li>
|
||||
<li><a href="._week38-bs021.html">22</a></li>
|
||||
<li><a href="._week38-bs022.html">23</a></li>
|
||||
<li class="active"><a href="._week38-bs023.html">24</a></li>
|
||||
<li><a href="._week38-bs024.html">25</a></li>
|
||||
<li><a href="._week38-bs025.html">26</a></li>
|
||||
<li><a href="._week38-bs026.html">27</a></li>
|
||||
<li><a href="._week38-bs027.html">28</a></li>
|
||||
<li><a href="._week38-bs028.html">29</a></li>
|
||||
<li><a href="._week38-bs029.html">30</a></li>
|
||||
<li><a href="._week38-bs030.html">31</a></li>
|
||||
<li><a href="._week38-bs031.html">32</a></li>
|
||||
<li><a href="._week38-bs032.html">33</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs039.html">40</a></li>
|
||||
<li><a href="._week38-bs024.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
||||
('Thursday September 17', 2, None, '___sec1'),
|
||||
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
||||
('Various steps in cross-validation', 2, None, '___sec3'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Cross-validation in brief', 2, None, '___sec5'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Cross-validation with Ridge', 2, None, '___sec9'),
|
||||
('The Ising model', 2, None, '___sec10'),
|
||||
('Reformulating the problem to suit regression',
|
||||
2,
|
||||
None,
|
||||
'___sec11'),
|
||||
('Linear regression', 2, None, '___sec12'),
|
||||
('Singular Value decomposition', 2, None, '___sec13'),
|
||||
('The one-dimensional Ising model', 2, None, '___sec14'),
|
||||
('Ridge regression', 2, None, '___sec15'),
|
||||
('LASSO regression', 2, None, '___sec16'),
|
||||
('Performance as function of the regularization parameter',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
||||
('Friday September 18: Intro to Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Logistic Regression', 2, None, '___sec20'),
|
||||
('Classification problems', 2, None, '___sec21'),
|
||||
('Optimization and Deep learning', 2, None, '___sec22'),
|
||||
('Basics', 2, None, '___sec23'),
|
||||
('Linear classifier', 2, None, '___sec24'),
|
||||
('Some selected properties', 2, None, '___sec25'),
|
||||
('The logistic function', 2, None, '___sec26'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'___sec27'),
|
||||
('Two parameters', 2, None, '___sec28'),
|
||||
('Maximum likelihood', 2, None, '___sec29'),
|
||||
('The cost function rewritten', 2, None, '___sec30'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec31'),
|
||||
('A more compact expression', 2, None, '___sec32'),
|
||||
('Extending to more predictors', 2, None, '___sec33'),
|
||||
('Including more classes', 2, None, '___sec34'),
|
||||
('More classes', 2, None, '___sec35'),
|
||||
('A simple classification problem', 2, None, '___sec36'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec37'),
|
||||
('Other measures in classification studies: Cancer Data again',
|
||||
2,
|
||||
None,
|
||||
'___sec38')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs005.html#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs023.html#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec23" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs025.html#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs026.html#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs027.html#___sec26" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs028.html#___sec27" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs029.html#___sec28" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs030.html#___sec29" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs031.html#___sec30" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs032.html#___sec31" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs033.html#___sec32" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs034.html#___sec33" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs035.html#___sec34" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs036.html#___sec35" style="font-size: 80%;">More classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs037.html#___sec36" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs038.html#___sec37" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs039.html#___sec38" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0024"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec23" class="anchor">Basics </h2>
|
||||
|
||||
<p>
|
||||
We consider the case where the dependent variables, also called the
|
||||
responses or the outcomes, \( y_i \) are discrete and only take values
|
||||
from \( k=0,\dots,K-1 \) (i.e. \( K \) classes).
|
||||
|
||||
<p>
|
||||
The goal is to predict the
|
||||
output classes from the design matrix \( \hat{X}\in\mathbb{R}^{n\times p} \)
|
||||
made of \( n \) samples, each of which carries \( p \) features or predictors. The
|
||||
primary goal is to identify the classes to which new unseen samples
|
||||
belong.
|
||||
|
||||
<p>
|
||||
Let us specialize to the case of two classes only, with outputs
|
||||
\( y_i=0 \) and \( y_i=1 \). Our outcomes could represent the status of a
|
||||
credit card user that could default or not on her/his credit card
|
||||
debt. That is
|
||||
|
||||
$$
|
||||
y_i = \begin{bmatrix} 0 & \mathrm{no}\\ 1 & \mathrm{yes} \end{bmatrix}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week38-bs023.html">«</a></li>
|
||||
<li><a href="._week38-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs016.html">17</a></li>
|
||||
<li><a href="._week38-bs017.html">18</a></li>
|
||||
<li><a href="._week38-bs018.html">19</a></li>
|
||||
<li><a href="._week38-bs019.html">20</a></li>
|
||||
<li><a href="._week38-bs020.html">21</a></li>
|
||||
<li><a href="._week38-bs021.html">22</a></li>
|
||||
<li><a href="._week38-bs022.html">23</a></li>
|
||||
<li><a href="._week38-bs023.html">24</a></li>
|
||||
<li class="active"><a href="._week38-bs024.html">25</a></li>
|
||||
<li><a href="._week38-bs025.html">26</a></li>
|
||||
<li><a href="._week38-bs026.html">27</a></li>
|
||||
<li><a href="._week38-bs027.html">28</a></li>
|
||||
<li><a href="._week38-bs028.html">29</a></li>
|
||||
<li><a href="._week38-bs029.html">30</a></li>
|
||||
<li><a href="._week38-bs030.html">31</a></li>
|
||||
<li><a href="._week38-bs031.html">32</a></li>
|
||||
<li><a href="._week38-bs032.html">33</a></li>
|
||||
<li><a href="._week38-bs033.html">34</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs039.html">40</a></li>
|
||||
<li><a href="._week38-bs025.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
||||
('Thursday September 17', 2, None, '___sec1'),
|
||||
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
||||
('Various steps in cross-validation', 2, None, '___sec3'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Cross-validation in brief', 2, None, '___sec5'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Cross-validation with Ridge', 2, None, '___sec9'),
|
||||
('The Ising model', 2, None, '___sec10'),
|
||||
('Reformulating the problem to suit regression',
|
||||
2,
|
||||
None,
|
||||
'___sec11'),
|
||||
('Linear regression', 2, None, '___sec12'),
|
||||
('Singular Value decomposition', 2, None, '___sec13'),
|
||||
('The one-dimensional Ising model', 2, None, '___sec14'),
|
||||
('Ridge regression', 2, None, '___sec15'),
|
||||
('LASSO regression', 2, None, '___sec16'),
|
||||
('Performance as function of the regularization parameter',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
||||
('Friday September 18: Intro to Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Logistic Regression', 2, None, '___sec20'),
|
||||
('Classification problems', 2, None, '___sec21'),
|
||||
('Optimization and Deep learning', 2, None, '___sec22'),
|
||||
('Basics', 2, None, '___sec23'),
|
||||
('Linear classifier', 2, None, '___sec24'),
|
||||
('Some selected properties', 2, None, '___sec25'),
|
||||
('The logistic function', 2, None, '___sec26'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'___sec27'),
|
||||
('Two parameters', 2, None, '___sec28'),
|
||||
('Maximum likelihood', 2, None, '___sec29'),
|
||||
('The cost function rewritten', 2, None, '___sec30'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec31'),
|
||||
('A more compact expression', 2, None, '___sec32'),
|
||||
('Extending to more predictors', 2, None, '___sec33'),
|
||||
('Including more classes', 2, None, '___sec34'),
|
||||
('More classes', 2, None, '___sec35'),
|
||||
('A simple classification problem', 2, None, '___sec36'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec37'),
|
||||
('Other measures in classification studies: Cancer Data again',
|
||||
2,
|
||||
None,
|
||||
'___sec38')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs005.html#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs023.html#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs024.html#___sec23" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs026.html#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs027.html#___sec26" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs028.html#___sec27" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs029.html#___sec28" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs030.html#___sec29" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs031.html#___sec30" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs032.html#___sec31" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs033.html#___sec32" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs034.html#___sec33" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs035.html#___sec34" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs036.html#___sec35" style="font-size: 80%;">More classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs037.html#___sec36" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs038.html#___sec37" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs039.html#___sec38" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0025"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec24" class="anchor">Linear classifier </h2>
|
||||
|
||||
<p>
|
||||
Before moving to the logistic model, let us try to use our linear
|
||||
regression model to classify these two outcomes. We could for example
|
||||
fit a linear model to the default case if \( y_i > 0.5 \) and the no
|
||||
default case \( y_i \leq 0.5 \).
|
||||
|
||||
<p>
|
||||
We would then have our
|
||||
weighted linear combination, namely
|
||||
$$
|
||||
\begin{equation}
|
||||
\hat{y} = \hat{X}^T\hat{\beta} + \hat{\epsilon},
|
||||
\tag{13}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
where \( \hat{y} \) is a vector representing the possible outcomes, \( \hat{X} \) is our
|
||||
\( n\times p \) design matrix and \( \hat{\beta} \) represents our estimators/predictors.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week38-bs024.html">«</a></li>
|
||||
<li><a href="._week38-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs017.html">18</a></li>
|
||||
<li><a href="._week38-bs018.html">19</a></li>
|
||||
<li><a href="._week38-bs019.html">20</a></li>
|
||||
<li><a href="._week38-bs020.html">21</a></li>
|
||||
<li><a href="._week38-bs021.html">22</a></li>
|
||||
<li><a href="._week38-bs022.html">23</a></li>
|
||||
<li><a href="._week38-bs023.html">24</a></li>
|
||||
<li><a href="._week38-bs024.html">25</a></li>
|
||||
<li class="active"><a href="._week38-bs025.html">26</a></li>
|
||||
<li><a href="._week38-bs026.html">27</a></li>
|
||||
<li><a href="._week38-bs027.html">28</a></li>
|
||||
<li><a href="._week38-bs028.html">29</a></li>
|
||||
<li><a href="._week38-bs029.html">30</a></li>
|
||||
<li><a href="._week38-bs030.html">31</a></li>
|
||||
<li><a href="._week38-bs031.html">32</a></li>
|
||||
<li><a href="._week38-bs032.html">33</a></li>
|
||||
<li><a href="._week38-bs033.html">34</a></li>
|
||||
<li><a href="._week38-bs034.html">35</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs039.html">40</a></li>
|
||||
<li><a href="._week38-bs026.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
||||
('Thursday September 17', 2, None, '___sec1'),
|
||||
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
||||
('Various steps in cross-validation', 2, None, '___sec3'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Cross-validation in brief', 2, None, '___sec5'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Cross-validation with Ridge', 2, None, '___sec9'),
|
||||
('The Ising model', 2, None, '___sec10'),
|
||||
('Reformulating the problem to suit regression',
|
||||
2,
|
||||
None,
|
||||
'___sec11'),
|
||||
('Linear regression', 2, None, '___sec12'),
|
||||
('Singular Value decomposition', 2, None, '___sec13'),
|
||||
('The one-dimensional Ising model', 2, None, '___sec14'),
|
||||
('Ridge regression', 2, None, '___sec15'),
|
||||
('LASSO regression', 2, None, '___sec16'),
|
||||
('Performance as function of the regularization parameter',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
||||
('Friday September 18: Intro to Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Logistic Regression', 2, None, '___sec20'),
|
||||
('Classification problems', 2, None, '___sec21'),
|
||||
('Optimization and Deep learning', 2, None, '___sec22'),
|
||||
('Basics', 2, None, '___sec23'),
|
||||
('Linear classifier', 2, None, '___sec24'),
|
||||
('Some selected properties', 2, None, '___sec25'),
|
||||
('The logistic function', 2, None, '___sec26'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'___sec27'),
|
||||
('Two parameters', 2, None, '___sec28'),
|
||||
('Maximum likelihood', 2, None, '___sec29'),
|
||||
('The cost function rewritten', 2, None, '___sec30'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec31'),
|
||||
('A more compact expression', 2, None, '___sec32'),
|
||||
('Extending to more predictors', 2, None, '___sec33'),
|
||||
('Including more classes', 2, None, '___sec34'),
|
||||
('More classes', 2, None, '___sec35'),
|
||||
('A simple classification problem', 2, None, '___sec36'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec37'),
|
||||
('Other measures in classification studies: Cancer Data again',
|
||||
2,
|
||||
None,
|
||||
'___sec38')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs005.html#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs023.html#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs024.html#___sec23" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs025.html#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs027.html#___sec26" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs028.html#___sec27" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs029.html#___sec28" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs030.html#___sec29" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs031.html#___sec30" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs032.html#___sec31" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs033.html#___sec32" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs034.html#___sec33" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs035.html#___sec34" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs036.html#___sec35" style="font-size: 80%;">More classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs037.html#___sec36" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs038.html#___sec37" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs039.html#___sec38" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0026"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec25" class="anchor">Some selected properties </h2>
|
||||
|
||||
<p>
|
||||
The main problem with our function is that it takes values on the
|
||||
entire real axis. In the case of logistic regression, however, the
|
||||
labels \( y_i \) are discrete variables. A typical example is the credit
|
||||
card data discussed below here, where we can set the state of
|
||||
defaulting the debt to \( y_i=1 \) and not to \( y_i=0 \) for one the persons
|
||||
in the data set (see the full example below).
|
||||
|
||||
<p>
|
||||
One simple way to get a discrete output is to have sign
|
||||
functions that map the output of a linear regressor to values \( \{0,1\} \),
|
||||
\( f(s_i)=sign(s_i)=1 \) if \( s_i\ge 0 \) and 0 if otherwise.
|
||||
We will encounter this model in our first demonstration of neural networks. Historically it is called the "perceptron" model in the machine learning
|
||||
literature. This model is extremely simple. However, in many cases it is more
|
||||
favorable to use a ``soft" classifier that outputs
|
||||
the probability of a given category. This leads us to the logistic function.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week38-bs025.html">«</a></li>
|
||||
<li><a href="._week38-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs018.html">19</a></li>
|
||||
<li><a href="._week38-bs019.html">20</a></li>
|
||||
<li><a href="._week38-bs020.html">21</a></li>
|
||||
<li><a href="._week38-bs021.html">22</a></li>
|
||||
<li><a href="._week38-bs022.html">23</a></li>
|
||||
<li><a href="._week38-bs023.html">24</a></li>
|
||||
<li><a href="._week38-bs024.html">25</a></li>
|
||||
<li><a href="._week38-bs025.html">26</a></li>
|
||||
<li class="active"><a href="._week38-bs026.html">27</a></li>
|
||||
<li><a href="._week38-bs027.html">28</a></li>
|
||||
<li><a href="._week38-bs028.html">29</a></li>
|
||||
<li><a href="._week38-bs029.html">30</a></li>
|
||||
<li><a href="._week38-bs030.html">31</a></li>
|
||||
<li><a href="._week38-bs031.html">32</a></li>
|
||||
<li><a href="._week38-bs032.html">33</a></li>
|
||||
<li><a href="._week38-bs033.html">34</a></li>
|
||||
<li><a href="._week38-bs034.html">35</a></li>
|
||||
<li><a href="._week38-bs035.html">36</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs039.html">40</a></li>
|
||||
<li><a href="._week38-bs027.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
||||
('Thursday September 17', 2, None, '___sec1'),
|
||||
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
||||
('Various steps in cross-validation', 2, None, '___sec3'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Cross-validation in brief', 2, None, '___sec5'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Cross-validation with Ridge', 2, None, '___sec9'),
|
||||
('The Ising model', 2, None, '___sec10'),
|
||||
('Reformulating the problem to suit regression',
|
||||
2,
|
||||
None,
|
||||
'___sec11'),
|
||||
('Linear regression', 2, None, '___sec12'),
|
||||
('Singular Value decomposition', 2, None, '___sec13'),
|
||||
('The one-dimensional Ising model', 2, None, '___sec14'),
|
||||
('Ridge regression', 2, None, '___sec15'),
|
||||
('LASSO regression', 2, None, '___sec16'),
|
||||
('Performance as function of the regularization parameter',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
||||
('Friday September 18: Intro to Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Logistic Regression', 2, None, '___sec20'),
|
||||
('Classification problems', 2, None, '___sec21'),
|
||||
('Optimization and Deep learning', 2, None, '___sec22'),
|
||||
('Basics', 2, None, '___sec23'),
|
||||
('Linear classifier', 2, None, '___sec24'),
|
||||
('Some selected properties', 2, None, '___sec25'),
|
||||
('The logistic function', 2, None, '___sec26'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'___sec27'),
|
||||
('Two parameters', 2, None, '___sec28'),
|
||||
('Maximum likelihood', 2, None, '___sec29'),
|
||||
('The cost function rewritten', 2, None, '___sec30'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec31'),
|
||||
('A more compact expression', 2, None, '___sec32'),
|
||||
('Extending to more predictors', 2, None, '___sec33'),
|
||||
('Including more classes', 2, None, '___sec34'),
|
||||
('More classes', 2, None, '___sec35'),
|
||||
('A simple classification problem', 2, None, '___sec36'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec37'),
|
||||
('Other measures in classification studies: Cancer Data again',
|
||||
2,
|
||||
None,
|
||||
'___sec38')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs005.html#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs023.html#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs024.html#___sec23" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs025.html#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs026.html#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec26" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs028.html#___sec27" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs029.html#___sec28" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs030.html#___sec29" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs031.html#___sec30" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs032.html#___sec31" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs033.html#___sec32" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs034.html#___sec33" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs035.html#___sec34" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs036.html#___sec35" style="font-size: 80%;">More classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs037.html#___sec36" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs038.html#___sec37" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs039.html#___sec38" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0027"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec26" class="anchor">The logistic function </h2>
|
||||
|
||||
<p>
|
||||
The perceptron is an example of a ``hard classification" model. We
|
||||
will encounter this model when we discuss neural networks as
|
||||
well. Each datapoint is deterministically assigned to a category (i.e
|
||||
\( y_i=0 \) or \( y_i=1 \)). In many cases, it is favorable to have a "soft"
|
||||
classifier that outputs the probability of a given category rather
|
||||
than a single value. For example, given \( x_i \), the classifier
|
||||
outputs the probability of being in a category \( k \). Logistic regression
|
||||
is the most common example of a so-called soft classifier. In logistic
|
||||
regression, the probability that a data point \( x_i \)
|
||||
belongs to a category \( y_i=\{0,1\} \) is given by the so-called logit function (or Sigmoid) which is meant to represent the likelihood for a given event,
|
||||
$$
|
||||
p(t) = \frac{1}{1+\mathrm \exp{-t}}=\frac{\exp{t}}{1+\mathrm \exp{t}}.
|
||||
$$
|
||||
|
||||
Note that \( 1-p(t)= p(-t) \).
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week38-bs026.html">«</a></li>
|
||||
<li><a href="._week38-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs019.html">20</a></li>
|
||||
<li><a href="._week38-bs020.html">21</a></li>
|
||||
<li><a href="._week38-bs021.html">22</a></li>
|
||||
<li><a href="._week38-bs022.html">23</a></li>
|
||||
<li><a href="._week38-bs023.html">24</a></li>
|
||||
<li><a href="._week38-bs024.html">25</a></li>
|
||||
<li><a href="._week38-bs025.html">26</a></li>
|
||||
<li><a href="._week38-bs026.html">27</a></li>
|
||||
<li class="active"><a href="._week38-bs027.html">28</a></li>
|
||||
<li><a href="._week38-bs028.html">29</a></li>
|
||||
<li><a href="._week38-bs029.html">30</a></li>
|
||||
<li><a href="._week38-bs030.html">31</a></li>
|
||||
<li><a href="._week38-bs031.html">32</a></li>
|
||||
<li><a href="._week38-bs032.html">33</a></li>
|
||||
<li><a href="._week38-bs033.html">34</a></li>
|
||||
<li><a href="._week38-bs034.html">35</a></li>
|
||||
<li><a href="._week38-bs035.html">36</a></li>
|
||||
<li><a href="._week38-bs036.html">37</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs039.html">40</a></li>
|
||||
<li><a href="._week38-bs028.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,317 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
||||
('Thursday September 17', 2, None, '___sec1'),
|
||||
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
||||
('Various steps in cross-validation', 2, None, '___sec3'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Cross-validation in brief', 2, None, '___sec5'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Cross-validation with Ridge', 2, None, '___sec9'),
|
||||
('The Ising model', 2, None, '___sec10'),
|
||||
('Reformulating the problem to suit regression',
|
||||
2,
|
||||
None,
|
||||
'___sec11'),
|
||||
('Linear regression', 2, None, '___sec12'),
|
||||
('Singular Value decomposition', 2, None, '___sec13'),
|
||||
('The one-dimensional Ising model', 2, None, '___sec14'),
|
||||
('Ridge regression', 2, None, '___sec15'),
|
||||
('LASSO regression', 2, None, '___sec16'),
|
||||
('Performance as function of the regularization parameter',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
||||
('Friday September 18: Intro to Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Logistic Regression', 2, None, '___sec20'),
|
||||
('Classification problems', 2, None, '___sec21'),
|
||||
('Optimization and Deep learning', 2, None, '___sec22'),
|
||||
('Basics', 2, None, '___sec23'),
|
||||
('Linear classifier', 2, None, '___sec24'),
|
||||
('Some selected properties', 2, None, '___sec25'),
|
||||
('The logistic function', 2, None, '___sec26'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'___sec27'),
|
||||
('Two parameters', 2, None, '___sec28'),
|
||||
('Maximum likelihood', 2, None, '___sec29'),
|
||||
('The cost function rewritten', 2, None, '___sec30'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec31'),
|
||||
('A more compact expression', 2, None, '___sec32'),
|
||||
('Extending to more predictors', 2, None, '___sec33'),
|
||||
('Including more classes', 2, None, '___sec34'),
|
||||
('More classes', 2, None, '___sec35'),
|
||||
('A simple classification problem', 2, None, '___sec36'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec37'),
|
||||
('Other measures in classification studies: Cancer Data again',
|
||||
2,
|
||||
None,
|
||||
'___sec38')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs005.html#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs023.html#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs024.html#___sec23" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs025.html#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs026.html#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs027.html#___sec26" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec27" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs029.html#___sec28" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs030.html#___sec29" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs031.html#___sec30" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs032.html#___sec31" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs033.html#___sec32" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs034.html#___sec33" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs035.html#___sec34" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs036.html#___sec35" style="font-size: 80%;">More classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs037.html#___sec36" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs038.html#___sec37" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs039.html#___sec38" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0028"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec27" class="anchor">Examples of likelihood functions used in logistic regression and nueral networks </h2>
|
||||
|
||||
<p>
|
||||
The following code plots the logistic function, the step function and other functions we will encounter from here and on.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #BA2121; font-style: italic">"""The sigmoid function (or the logistic curve) is a</span>
|
||||
<span style="color: #BA2121; font-style: italic">function that takes any real number, z, and outputs a number (0,1).</span>
|
||||
<span style="color: #BA2121; font-style: italic">It is useful in neural networks for assigning weights on a relative scale.</span>
|
||||
<span style="color: #BA2121; font-style: italic">The value z is the weighted sum of parameters involved in the learning algorithm."""</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">math</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">mt</span>
|
||||
|
||||
z <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>arange(<span style="color: #666666">-5</span>, <span style="color: #666666">5</span>, <span style="color: #666666">.1</span>)
|
||||
sigma_fn <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>vectorize(<span style="color: #008000; font-weight: bold">lambda</span> z: <span style="color: #666666">1/</span>(<span style="color: #666666">1+</span>numpy<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>z)))
|
||||
sigma <span style="color: #666666">=</span> sigma_fn(z)
|
||||
|
||||
fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
|
||||
ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot(<span style="color: #666666">111</span>)
|
||||
ax<span style="color: #666666">.</span>plot(z, sigma)
|
||||
ax<span style="color: #666666">.</span>set_ylim([<span style="color: #666666">-0.1</span>, <span style="color: #666666">1.1</span>])
|
||||
ax<span style="color: #666666">.</span>set_xlim([<span style="color: #666666">-5</span>,<span style="color: #666666">5</span>])
|
||||
ax<span style="color: #666666">.</span>grid(<span style="color: #008000">True</span>)
|
||||
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">'z'</span>)
|
||||
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">'sigmoid function'</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
<span style="color: #BA2121; font-style: italic">"""Step Function"""</span>
|
||||
z <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>arange(<span style="color: #666666">-5</span>, <span style="color: #666666">5</span>, <span style="color: #666666">.02</span>)
|
||||
step_fn <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>vectorize(<span style="color: #008000; font-weight: bold">lambda</span> z: <span style="color: #666666">1.0</span> <span style="color: #008000; font-weight: bold">if</span> z <span style="color: #666666">>=</span> <span style="color: #666666">0.0</span> <span style="color: #008000; font-weight: bold">else</span> <span style="color: #666666">0.0</span>)
|
||||
step <span style="color: #666666">=</span> step_fn(z)
|
||||
|
||||
fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
|
||||
ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot(<span style="color: #666666">111</span>)
|
||||
ax<span style="color: #666666">.</span>plot(z, step)
|
||||
ax<span style="color: #666666">.</span>set_ylim([<span style="color: #666666">-0.5</span>, <span style="color: #666666">1.5</span>])
|
||||
ax<span style="color: #666666">.</span>set_xlim([<span style="color: #666666">-5</span>,<span style="color: #666666">5</span>])
|
||||
ax<span style="color: #666666">.</span>grid(<span style="color: #008000">True</span>)
|
||||
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">'z'</span>)
|
||||
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">'step function'</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
<span style="color: #BA2121; font-style: italic">"""tanh Function"""</span>
|
||||
z <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>arange(<span style="color: #666666">-2*</span>mt<span style="color: #666666">.</span>pi, <span style="color: #666666">2*</span>mt<span style="color: #666666">.</span>pi, <span style="color: #666666">0.1</span>)
|
||||
t <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>tanh(z)
|
||||
|
||||
fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
|
||||
ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot(<span style="color: #666666">111</span>)
|
||||
ax<span style="color: #666666">.</span>plot(z, t)
|
||||
ax<span style="color: #666666">.</span>set_ylim([<span style="color: #666666">-1.0</span>, <span style="color: #666666">1.0</span>])
|
||||
ax<span style="color: #666666">.</span>set_xlim([<span style="color: #666666">-2*</span>mt<span style="color: #666666">.</span>pi,<span style="color: #666666">2*</span>mt<span style="color: #666666">.</span>pi])
|
||||
ax<span style="color: #666666">.</span>grid(<span style="color: #008000">True</span>)
|
||||
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">'z'</span>)
|
||||
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">'tanh function'</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week38-bs027.html">«</a></li>
|
||||
<li><a href="._week38-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs020.html">21</a></li>
|
||||
<li><a href="._week38-bs021.html">22</a></li>
|
||||
<li><a href="._week38-bs022.html">23</a></li>
|
||||
<li><a href="._week38-bs023.html">24</a></li>
|
||||
<li><a href="._week38-bs024.html">25</a></li>
|
||||
<li><a href="._week38-bs025.html">26</a></li>
|
||||
<li><a href="._week38-bs026.html">27</a></li>
|
||||
<li><a href="._week38-bs027.html">28</a></li>
|
||||
<li class="active"><a href="._week38-bs028.html">29</a></li>
|
||||
<li><a href="._week38-bs029.html">30</a></li>
|
||||
<li><a href="._week38-bs030.html">31</a></li>
|
||||
<li><a href="._week38-bs031.html">32</a></li>
|
||||
<li><a href="._week38-bs032.html">33</a></li>
|
||||
<li><a href="._week38-bs033.html">34</a></li>
|
||||
<li><a href="._week38-bs034.html">35</a></li>
|
||||
<li><a href="._week38-bs035.html">36</a></li>
|
||||
<li><a href="._week38-bs036.html">37</a></li>
|
||||
<li><a href="._week38-bs037.html">38</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs039.html">40</a></li>
|
||||
<li><a href="._week38-bs029.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
||||
('Thursday September 17', 2, None, '___sec1'),
|
||||
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
||||
('Various steps in cross-validation', 2, None, '___sec3'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Cross-validation in brief', 2, None, '___sec5'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Cross-validation with Ridge', 2, None, '___sec9'),
|
||||
('The Ising model', 2, None, '___sec10'),
|
||||
('Reformulating the problem to suit regression',
|
||||
2,
|
||||
None,
|
||||
'___sec11'),
|
||||
('Linear regression', 2, None, '___sec12'),
|
||||
('Singular Value decomposition', 2, None, '___sec13'),
|
||||
('The one-dimensional Ising model', 2, None, '___sec14'),
|
||||
('Ridge regression', 2, None, '___sec15'),
|
||||
('LASSO regression', 2, None, '___sec16'),
|
||||
('Performance as function of the regularization parameter',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
||||
('Friday September 18: Intro to Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Logistic Regression', 2, None, '___sec20'),
|
||||
('Classification problems', 2, None, '___sec21'),
|
||||
('Optimization and Deep learning', 2, None, '___sec22'),
|
||||
('Basics', 2, None, '___sec23'),
|
||||
('Linear classifier', 2, None, '___sec24'),
|
||||
('Some selected properties', 2, None, '___sec25'),
|
||||
('The logistic function', 2, None, '___sec26'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'___sec27'),
|
||||
('Two parameters', 2, None, '___sec28'),
|
||||
('Maximum likelihood', 2, None, '___sec29'),
|
||||
('The cost function rewritten', 2, None, '___sec30'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec31'),
|
||||
('A more compact expression', 2, None, '___sec32'),
|
||||
('Extending to more predictors', 2, None, '___sec33'),
|
||||
('Including more classes', 2, None, '___sec34'),
|
||||
('More classes', 2, None, '___sec35'),
|
||||
('A simple classification problem', 2, None, '___sec36'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec37'),
|
||||
('Other measures in classification studies: Cancer Data again',
|
||||
2,
|
||||
None,
|
||||
'___sec38')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs005.html#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs023.html#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs024.html#___sec23" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs025.html#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs026.html#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs027.html#___sec26" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs028.html#___sec27" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec28" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs030.html#___sec29" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs031.html#___sec30" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs032.html#___sec31" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs033.html#___sec32" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs034.html#___sec33" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs035.html#___sec34" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs036.html#___sec35" style="font-size: 80%;">More classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs037.html#___sec36" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs038.html#___sec37" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs039.html#___sec38" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0029"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec28" class="anchor">Two parameters </h2>
|
||||
|
||||
<p>
|
||||
We assume now that we have two classes with \( y_i \) either \( 0 \) or \( 1 \). Furthermore we assume also that we have only two parameters \( \beta \) in our fitting of the Sigmoid function, that is we define probabilities
|
||||
$$
|
||||
\begin{align*}
|
||||
p(y_i=1|x_i,\hat{\beta}) &= \frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}},\nonumber\\
|
||||
p(y_i=0|x_i,\hat{\beta}) &= 1 - p(y_i=1|x_i,\hat{\beta}),
|
||||
\end{align*}
|
||||
$$
|
||||
|
||||
where \( \hat{\beta} \) are the weights we wish to extract from data, in our case \( \beta_0 \) and \( \beta_1 \).
|
||||
|
||||
<p>
|
||||
Note that we used
|
||||
$$
|
||||
p(y_i=0\vert x_i, \hat{\beta}) = 1-p(y_i=1\vert x_i, \hat{\beta}).
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week38-bs028.html">«</a></li>
|
||||
<li><a href="._week38-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs021.html">22</a></li>
|
||||
<li><a href="._week38-bs022.html">23</a></li>
|
||||
<li><a href="._week38-bs023.html">24</a></li>
|
||||
<li><a href="._week38-bs024.html">25</a></li>
|
||||
<li><a href="._week38-bs025.html">26</a></li>
|
||||
<li><a href="._week38-bs026.html">27</a></li>
|
||||
<li><a href="._week38-bs027.html">28</a></li>
|
||||
<li><a href="._week38-bs028.html">29</a></li>
|
||||
<li class="active"><a href="._week38-bs029.html">30</a></li>
|
||||
<li><a href="._week38-bs030.html">31</a></li>
|
||||
<li><a href="._week38-bs031.html">32</a></li>
|
||||
<li><a href="._week38-bs032.html">33</a></li>
|
||||
<li><a href="._week38-bs033.html">34</a></li>
|
||||
<li><a href="._week38-bs034.html">35</a></li>
|
||||
<li><a href="._week38-bs035.html">36</a></li>
|
||||
<li><a href="._week38-bs036.html">37</a></li>
|
||||
<li><a href="._week38-bs037.html">38</a></li>
|
||||
<li><a href="._week38-bs038.html">39</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs039.html">40</a></li>
|
||||
<li><a href="._week38-bs030.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
||||
('Thursday September 17', 2, None, '___sec1'),
|
||||
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
||||
('Various steps in cross-validation', 2, None, '___sec3'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Cross-validation in brief', 2, None, '___sec5'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Cross-validation with Ridge', 2, None, '___sec9'),
|
||||
('The Ising model', 2, None, '___sec10'),
|
||||
('Reformulating the problem to suit regression',
|
||||
2,
|
||||
None,
|
||||
'___sec11'),
|
||||
('Linear regression', 2, None, '___sec12'),
|
||||
('Singular Value decomposition', 2, None, '___sec13'),
|
||||
('The one-dimensional Ising model', 2, None, '___sec14'),
|
||||
('Ridge regression', 2, None, '___sec15'),
|
||||
('LASSO regression', 2, None, '___sec16'),
|
||||
('Performance as function of the regularization parameter',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
||||
('Friday September 18: Intro to Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Logistic Regression', 2, None, '___sec20'),
|
||||
('Classification problems', 2, None, '___sec21'),
|
||||
('Optimization and Deep learning', 2, None, '___sec22'),
|
||||
('Basics', 2, None, '___sec23'),
|
||||
('Linear classifier', 2, None, '___sec24'),
|
||||
('Some selected properties', 2, None, '___sec25'),
|
||||
('The logistic function', 2, None, '___sec26'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'___sec27'),
|
||||
('Two parameters', 2, None, '___sec28'),
|
||||
('Maximum likelihood', 2, None, '___sec29'),
|
||||
('The cost function rewritten', 2, None, '___sec30'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec31'),
|
||||
('A more compact expression', 2, None, '___sec32'),
|
||||
('Extending to more predictors', 2, None, '___sec33'),
|
||||
('Including more classes', 2, None, '___sec34'),
|
||||
('More classes', 2, None, '___sec35'),
|
||||
('A simple classification problem', 2, None, '___sec36'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec37'),
|
||||
('Other measures in classification studies: Cancer Data again',
|
||||
2,
|
||||
None,
|
||||
'___sec38')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs005.html#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs023.html#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs024.html#___sec23" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs025.html#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs026.html#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs027.html#___sec26" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs028.html#___sec27" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs029.html#___sec28" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec29" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs031.html#___sec30" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs032.html#___sec31" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs033.html#___sec32" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs034.html#___sec33" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs035.html#___sec34" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs036.html#___sec35" style="font-size: 80%;">More classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs037.html#___sec36" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs038.html#___sec37" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs039.html#___sec38" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0030"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec29" class="anchor">Maximum likelihood </h2>
|
||||
|
||||
<p>
|
||||
In order to define the total likelihood for all possible outcomes from a
|
||||
dataset \( \mathcal{D}=\{(y_i,x_i)\} \), with the binary labels
|
||||
\( y_i\in\{0,1\} \) and where the data points are drawn independently, we use the so-called <a href="https://en.wikipedia.org/wiki/Maximum_likelihood_estimation" target="_self">Maximum Likelihood Estimation</a> (MLE) principle.
|
||||
We aim thus at maximizing
|
||||
the probability of seeing the observed data. We can then approximate the
|
||||
likelihood in terms of the product of the individual probabilities of a specific outcome \( y_i \), that is
|
||||
$$
|
||||
\begin{align*}
|
||||
P(\mathcal{D}|\hat{\beta})& = \prod_{i=1}^n \left[p(y_i=1|x_i,\hat{\beta})\right]^{y_i}\left[1-p(y_i=1|x_i,\hat{\beta}))\right]^{1-y_i}\nonumber \\
|
||||
\end{align*}
|
||||
$$
|
||||
|
||||
from which we obtain the log-likelihood and our <b>cost/loss</b> function
|
||||
$$
|
||||
\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n \left( y_i\log{p(y_i=1|x_i,\hat{\beta})} + (1-y_i)\log\left[1-p(y_i=1|x_i,\hat{\beta}))\right]\right).
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week38-bs029.html">«</a></li>
|
||||
<li><a href="._week38-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs022.html">23</a></li>
|
||||
<li><a href="._week38-bs023.html">24</a></li>
|
||||
<li><a href="._week38-bs024.html">25</a></li>
|
||||
<li><a href="._week38-bs025.html">26</a></li>
|
||||
<li><a href="._week38-bs026.html">27</a></li>
|
||||
<li><a href="._week38-bs027.html">28</a></li>
|
||||
<li><a href="._week38-bs028.html">29</a></li>
|
||||
<li><a href="._week38-bs029.html">30</a></li>
|
||||
<li class="active"><a href="._week38-bs030.html">31</a></li>
|
||||
<li><a href="._week38-bs031.html">32</a></li>
|
||||
<li><a href="._week38-bs032.html">33</a></li>
|
||||
<li><a href="._week38-bs033.html">34</a></li>
|
||||
<li><a href="._week38-bs034.html">35</a></li>
|
||||
<li><a href="._week38-bs035.html">36</a></li>
|
||||
<li><a href="._week38-bs036.html">37</a></li>
|
||||
<li><a href="._week38-bs037.html">38</a></li>
|
||||
<li><a href="._week38-bs038.html">39</a></li>
|
||||
<li><a href="._week38-bs039.html">40</a></li>
|
||||
<li><a href="._week38-bs031.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
||||
('Thursday September 17', 2, None, '___sec1'),
|
||||
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
||||
('Various steps in cross-validation', 2, None, '___sec3'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Cross-validation in brief', 2, None, '___sec5'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Cross-validation with Ridge', 2, None, '___sec9'),
|
||||
('The Ising model', 2, None, '___sec10'),
|
||||
('Reformulating the problem to suit regression',
|
||||
2,
|
||||
None,
|
||||
'___sec11'),
|
||||
('Linear regression', 2, None, '___sec12'),
|
||||
('Singular Value decomposition', 2, None, '___sec13'),
|
||||
('The one-dimensional Ising model', 2, None, '___sec14'),
|
||||
('Ridge regression', 2, None, '___sec15'),
|
||||
('LASSO regression', 2, None, '___sec16'),
|
||||
('Performance as function of the regularization parameter',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
||||
('Friday September 18: Intro to Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Logistic Regression', 2, None, '___sec20'),
|
||||
('Classification problems', 2, None, '___sec21'),
|
||||
('Optimization and Deep learning', 2, None, '___sec22'),
|
||||
('Basics', 2, None, '___sec23'),
|
||||
('Linear classifier', 2, None, '___sec24'),
|
||||
('Some selected properties', 2, None, '___sec25'),
|
||||
('The logistic function', 2, None, '___sec26'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'___sec27'),
|
||||
('Two parameters', 2, None, '___sec28'),
|
||||
('Maximum likelihood', 2, None, '___sec29'),
|
||||
('The cost function rewritten', 2, None, '___sec30'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec31'),
|
||||
('A more compact expression', 2, None, '___sec32'),
|
||||
('Extending to more predictors', 2, None, '___sec33'),
|
||||
('Including more classes', 2, None, '___sec34'),
|
||||
('More classes', 2, None, '___sec35'),
|
||||
('A simple classification problem', 2, None, '___sec36'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec37'),
|
||||
('Other measures in classification studies: Cancer Data again',
|
||||
2,
|
||||
None,
|
||||
'___sec38')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs005.html#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs023.html#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs024.html#___sec23" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs025.html#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs026.html#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs027.html#___sec26" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs028.html#___sec27" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs029.html#___sec28" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs030.html#___sec29" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec30" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs032.html#___sec31" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs033.html#___sec32" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs034.html#___sec33" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs035.html#___sec34" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs036.html#___sec35" style="font-size: 80%;">More classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs037.html#___sec36" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs038.html#___sec37" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs039.html#___sec38" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0031"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec30" class="anchor">The cost function rewritten </h2>
|
||||
|
||||
<p>
|
||||
Reordering the logarithms, we can rewrite the <b>cost/loss</b> function as
|
||||
$$
|
||||
\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{(1+\exp{(\beta_0+\beta_1x_i)})}\right).
|
||||
$$
|
||||
|
||||
<p>
|
||||
The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to \( \beta \).
|
||||
Since the cost (error) function is just the negative log-likelihood, for logistic regression we have that
|
||||
$$
|
||||
\mathcal{C}(\hat{\beta})=-\sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{(1+\exp{(\beta_0+\beta_1x_i)})}\right).
|
||||
$$
|
||||
|
||||
This equation is known in statistics as the <b>cross entropy</b>. Finally, we note that just as in linear regression,
|
||||
in practice we often supplement the cross-entropy with additional regularization terms, usually \( L_1 \) and \( L_2 \) regularization as we did for Ridge and Lasso regression.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week38-bs030.html">«</a></li>
|
||||
<li><a href="._week38-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs023.html">24</a></li>
|
||||
<li><a href="._week38-bs024.html">25</a></li>
|
||||
<li><a href="._week38-bs025.html">26</a></li>
|
||||
<li><a href="._week38-bs026.html">27</a></li>
|
||||
<li><a href="._week38-bs027.html">28</a></li>
|
||||
<li><a href="._week38-bs028.html">29</a></li>
|
||||
<li><a href="._week38-bs029.html">30</a></li>
|
||||
<li><a href="._week38-bs030.html">31</a></li>
|
||||
<li class="active"><a href="._week38-bs031.html">32</a></li>
|
||||
<li><a href="._week38-bs032.html">33</a></li>
|
||||
<li><a href="._week38-bs033.html">34</a></li>
|
||||
<li><a href="._week38-bs034.html">35</a></li>
|
||||
<li><a href="._week38-bs035.html">36</a></li>
|
||||
<li><a href="._week38-bs036.html">37</a></li>
|
||||
<li><a href="._week38-bs037.html">38</a></li>
|
||||
<li><a href="._week38-bs038.html">39</a></li>
|
||||
<li><a href="._week38-bs039.html">40</a></li>
|
||||
<li><a href="._week38-bs032.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
||||
('Thursday September 17', 2, None, '___sec1'),
|
||||
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
||||
('Various steps in cross-validation', 2, None, '___sec3'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Cross-validation in brief', 2, None, '___sec5'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Cross-validation with Ridge', 2, None, '___sec9'),
|
||||
('The Ising model', 2, None, '___sec10'),
|
||||
('Reformulating the problem to suit regression',
|
||||
2,
|
||||
None,
|
||||
'___sec11'),
|
||||
('Linear regression', 2, None, '___sec12'),
|
||||
('Singular Value decomposition', 2, None, '___sec13'),
|
||||
('The one-dimensional Ising model', 2, None, '___sec14'),
|
||||
('Ridge regression', 2, None, '___sec15'),
|
||||
('LASSO regression', 2, None, '___sec16'),
|
||||
('Performance as function of the regularization parameter',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
||||
('Friday September 18: Intro to Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Logistic Regression', 2, None, '___sec20'),
|
||||
('Classification problems', 2, None, '___sec21'),
|
||||
('Optimization and Deep learning', 2, None, '___sec22'),
|
||||
('Basics', 2, None, '___sec23'),
|
||||
('Linear classifier', 2, None, '___sec24'),
|
||||
('Some selected properties', 2, None, '___sec25'),
|
||||
('The logistic function', 2, None, '___sec26'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'___sec27'),
|
||||
('Two parameters', 2, None, '___sec28'),
|
||||
('Maximum likelihood', 2, None, '___sec29'),
|
||||
('The cost function rewritten', 2, None, '___sec30'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec31'),
|
||||
('A more compact expression', 2, None, '___sec32'),
|
||||
('Extending to more predictors', 2, None, '___sec33'),
|
||||
('Including more classes', 2, None, '___sec34'),
|
||||
('More classes', 2, None, '___sec35'),
|
||||
('A simple classification problem', 2, None, '___sec36'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec37'),
|
||||
('Other measures in classification studies: Cancer Data again',
|
||||
2,
|
||||
None,
|
||||
'___sec38')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs005.html#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs023.html#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs024.html#___sec23" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs025.html#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs026.html#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs027.html#___sec26" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs028.html#___sec27" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs029.html#___sec28" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs030.html#___sec29" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs031.html#___sec30" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec31" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs033.html#___sec32" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs034.html#___sec33" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs035.html#___sec34" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs036.html#___sec35" style="font-size: 80%;">More classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs037.html#___sec36" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs038.html#___sec37" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs039.html#___sec38" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0032"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec31" class="anchor">Minimizing the cross entropy </h2>
|
||||
|
||||
<p>
|
||||
The cross entropy is a convex function of the weights \( \hat{\beta} \) and,
|
||||
therefore, any local minimizer is a global minimizer.
|
||||
|
||||
<p>
|
||||
Minimizing this
|
||||
cost function with respect to the two parameters \( \beta_0 \) and \( \beta_1 \) we obtain
|
||||
|
||||
$$
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_0} = -\sum_{i=1}^n \left(y_i -\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right),
|
||||
$$
|
||||
|
||||
and
|
||||
$$
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_1} = -\sum_{i=1}^n \left(y_ix_i -x_i\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week38-bs031.html">«</a></li>
|
||||
<li><a href="._week38-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs024.html">25</a></li>
|
||||
<li><a href="._week38-bs025.html">26</a></li>
|
||||
<li><a href="._week38-bs026.html">27</a></li>
|
||||
<li><a href="._week38-bs027.html">28</a></li>
|
||||
<li><a href="._week38-bs028.html">29</a></li>
|
||||
<li><a href="._week38-bs029.html">30</a></li>
|
||||
<li><a href="._week38-bs030.html">31</a></li>
|
||||
<li><a href="._week38-bs031.html">32</a></li>
|
||||
<li class="active"><a href="._week38-bs032.html">33</a></li>
|
||||
<li><a href="._week38-bs033.html">34</a></li>
|
||||
<li><a href="._week38-bs034.html">35</a></li>
|
||||
<li><a href="._week38-bs035.html">36</a></li>
|
||||
<li><a href="._week38-bs036.html">37</a></li>
|
||||
<li><a href="._week38-bs037.html">38</a></li>
|
||||
<li><a href="._week38-bs038.html">39</a></li>
|
||||
<li><a href="._week38-bs039.html">40</a></li>
|
||||
<li><a href="._week38-bs033.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
||||
('Thursday September 17', 2, None, '___sec1'),
|
||||
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
||||
('Various steps in cross-validation', 2, None, '___sec3'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Cross-validation in brief', 2, None, '___sec5'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Cross-validation with Ridge', 2, None, '___sec9'),
|
||||
('The Ising model', 2, None, '___sec10'),
|
||||
('Reformulating the problem to suit regression',
|
||||
2,
|
||||
None,
|
||||
'___sec11'),
|
||||
('Linear regression', 2, None, '___sec12'),
|
||||
('Singular Value decomposition', 2, None, '___sec13'),
|
||||
('The one-dimensional Ising model', 2, None, '___sec14'),
|
||||
('Ridge regression', 2, None, '___sec15'),
|
||||
('LASSO regression', 2, None, '___sec16'),
|
||||
('Performance as function of the regularization parameter',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
||||
('Friday September 18: Intro to Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Logistic Regression', 2, None, '___sec20'),
|
||||
('Classification problems', 2, None, '___sec21'),
|
||||
('Optimization and Deep learning', 2, None, '___sec22'),
|
||||
('Basics', 2, None, '___sec23'),
|
||||
('Linear classifier', 2, None, '___sec24'),
|
||||
('Some selected properties', 2, None, '___sec25'),
|
||||
('The logistic function', 2, None, '___sec26'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'___sec27'),
|
||||
('Two parameters', 2, None, '___sec28'),
|
||||
('Maximum likelihood', 2, None, '___sec29'),
|
||||
('The cost function rewritten', 2, None, '___sec30'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec31'),
|
||||
('A more compact expression', 2, None, '___sec32'),
|
||||
('Extending to more predictors', 2, None, '___sec33'),
|
||||
('Including more classes', 2, None, '___sec34'),
|
||||
('More classes', 2, None, '___sec35'),
|
||||
('A simple classification problem', 2, None, '___sec36'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec37'),
|
||||
('Other measures in classification studies: Cancer Data again',
|
||||
2,
|
||||
None,
|
||||
'___sec38')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs005.html#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs023.html#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs024.html#___sec23" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs025.html#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs026.html#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs027.html#___sec26" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs028.html#___sec27" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs029.html#___sec28" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs030.html#___sec29" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs031.html#___sec30" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs032.html#___sec31" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec32" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs034.html#___sec33" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs035.html#___sec34" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs036.html#___sec35" style="font-size: 80%;">More classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs037.html#___sec36" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs038.html#___sec37" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs039.html#___sec38" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0033"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec32" class="anchor">A more compact expression </h2>
|
||||
|
||||
<p>
|
||||
Let us now define a vector \( \hat{y} \) with \( n \) elements \( y_i \), an
|
||||
\( n\times p \) matrix \( \hat{X} \) which contains the \( x_i \) values and a
|
||||
vector \( \hat{p} \) of fitted probabilities \( p(y_i\vert x_i,\hat{\beta}) \). We can rewrite in a more compact form the first
|
||||
derivative of cost function as
|
||||
|
||||
$$
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}} = -\hat{X}^T\left(\hat{y}-\hat{p}\right).
|
||||
$$
|
||||
|
||||
<p>
|
||||
If we in addition define a diagonal matrix \( \hat{W} \) with elements
|
||||
\( p(y_i\vert x_i,\hat{\beta})(1-p(y_i\vert x_i,\hat{\beta}) \), we can obtain a compact expression of the second derivative as
|
||||
|
||||
$$
|
||||
\frac{\partial^2 \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}\partial \hat{\beta}^T} = \hat{X}^T\hat{W}\hat{X}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week38-bs032.html">«</a></li>
|
||||
<li><a href="._week38-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs025.html">26</a></li>
|
||||
<li><a href="._week38-bs026.html">27</a></li>
|
||||
<li><a href="._week38-bs027.html">28</a></li>
|
||||
<li><a href="._week38-bs028.html">29</a></li>
|
||||
<li><a href="._week38-bs029.html">30</a></li>
|
||||
<li><a href="._week38-bs030.html">31</a></li>
|
||||
<li><a href="._week38-bs031.html">32</a></li>
|
||||
<li><a href="._week38-bs032.html">33</a></li>
|
||||
<li class="active"><a href="._week38-bs033.html">34</a></li>
|
||||
<li><a href="._week38-bs034.html">35</a></li>
|
||||
<li><a href="._week38-bs035.html">36</a></li>
|
||||
<li><a href="._week38-bs036.html">37</a></li>
|
||||
<li><a href="._week38-bs037.html">38</a></li>
|
||||
<li><a href="._week38-bs038.html">39</a></li>
|
||||
<li><a href="._week38-bs039.html">40</a></li>
|
||||
<li><a href="._week38-bs034.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
||||
('Thursday September 17', 2, None, '___sec1'),
|
||||
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
||||
('Various steps in cross-validation', 2, None, '___sec3'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Cross-validation in brief', 2, None, '___sec5'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Cross-validation with Ridge', 2, None, '___sec9'),
|
||||
('The Ising model', 2, None, '___sec10'),
|
||||
('Reformulating the problem to suit regression',
|
||||
2,
|
||||
None,
|
||||
'___sec11'),
|
||||
('Linear regression', 2, None, '___sec12'),
|
||||
('Singular Value decomposition', 2, None, '___sec13'),
|
||||
('The one-dimensional Ising model', 2, None, '___sec14'),
|
||||
('Ridge regression', 2, None, '___sec15'),
|
||||
('LASSO regression', 2, None, '___sec16'),
|
||||
('Performance as function of the regularization parameter',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
||||
('Friday September 18: Intro to Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Logistic Regression', 2, None, '___sec20'),
|
||||
('Classification problems', 2, None, '___sec21'),
|
||||
('Optimization and Deep learning', 2, None, '___sec22'),
|
||||
('Basics', 2, None, '___sec23'),
|
||||
('Linear classifier', 2, None, '___sec24'),
|
||||
('Some selected properties', 2, None, '___sec25'),
|
||||
('The logistic function', 2, None, '___sec26'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'___sec27'),
|
||||
('Two parameters', 2, None, '___sec28'),
|
||||
('Maximum likelihood', 2, None, '___sec29'),
|
||||
('The cost function rewritten', 2, None, '___sec30'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec31'),
|
||||
('A more compact expression', 2, None, '___sec32'),
|
||||
('Extending to more predictors', 2, None, '___sec33'),
|
||||
('Including more classes', 2, None, '___sec34'),
|
||||
('More classes', 2, None, '___sec35'),
|
||||
('A simple classification problem', 2, None, '___sec36'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec37'),
|
||||
('Other measures in classification studies: Cancer Data again',
|
||||
2,
|
||||
None,
|
||||
'___sec38')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs005.html#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs023.html#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs024.html#___sec23" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs025.html#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs026.html#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs027.html#___sec26" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs028.html#___sec27" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs029.html#___sec28" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs030.html#___sec29" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs031.html#___sec30" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs032.html#___sec31" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs033.html#___sec32" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec33" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs035.html#___sec34" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs036.html#___sec35" style="font-size: 80%;">More classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs037.html#___sec36" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs038.html#___sec37" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs039.html#___sec38" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0034"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec33" class="anchor">Extending to more predictors </h2>
|
||||
|
||||
<p>
|
||||
Within a binary classification problem, we can easily expand our model to include multiple predictors. Our ratio between likelihoods is then with \( p \) predictors
|
||||
$$
|
||||
\log{ \frac{p(\hat{\beta}\hat{x})}{1-p(\hat{\beta}\hat{x})}} = \beta_0+\beta_1x_1+\beta_2x_2+\dots+\beta_px_p.
|
||||
$$
|
||||
|
||||
Here we defined \( \hat{x}=[1,x_1,x_2,\dots,x_p] \) and \( \hat{\beta}=[\beta_0, \beta_1, \dots, \beta_p] \) leading to
|
||||
$$
|
||||
p(\hat{\beta}\hat{x})=\frac{ \exp{(\beta_0+\beta_1x_1+\beta_2x_2+\dots+\beta_px_p)}}{1+\exp{(\beta_0+\beta_1x_1+\beta_2x_2+\dots+\beta_px_p)}}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week38-bs033.html">«</a></li>
|
||||
<li><a href="._week38-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs026.html">27</a></li>
|
||||
<li><a href="._week38-bs027.html">28</a></li>
|
||||
<li><a href="._week38-bs028.html">29</a></li>
|
||||
<li><a href="._week38-bs029.html">30</a></li>
|
||||
<li><a href="._week38-bs030.html">31</a></li>
|
||||
<li><a href="._week38-bs031.html">32</a></li>
|
||||
<li><a href="._week38-bs032.html">33</a></li>
|
||||
<li><a href="._week38-bs033.html">34</a></li>
|
||||
<li class="active"><a href="._week38-bs034.html">35</a></li>
|
||||
<li><a href="._week38-bs035.html">36</a></li>
|
||||
<li><a href="._week38-bs036.html">37</a></li>
|
||||
<li><a href="._week38-bs037.html">38</a></li>
|
||||
<li><a href="._week38-bs038.html">39</a></li>
|
||||
<li><a href="._week38-bs039.html">40</a></li>
|
||||
<li><a href="._week38-bs035.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
||||
('Thursday September 17', 2, None, '___sec1'),
|
||||
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
||||
('Various steps in cross-validation', 2, None, '___sec3'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Cross-validation in brief', 2, None, '___sec5'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Cross-validation with Ridge', 2, None, '___sec9'),
|
||||
('The Ising model', 2, None, '___sec10'),
|
||||
('Reformulating the problem to suit regression',
|
||||
2,
|
||||
None,
|
||||
'___sec11'),
|
||||
('Linear regression', 2, None, '___sec12'),
|
||||
('Singular Value decomposition', 2, None, '___sec13'),
|
||||
('The one-dimensional Ising model', 2, None, '___sec14'),
|
||||
('Ridge regression', 2, None, '___sec15'),
|
||||
('LASSO regression', 2, None, '___sec16'),
|
||||
('Performance as function of the regularization parameter',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
||||
('Friday September 18: Intro to Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Logistic Regression', 2, None, '___sec20'),
|
||||
('Classification problems', 2, None, '___sec21'),
|
||||
('Optimization and Deep learning', 2, None, '___sec22'),
|
||||
('Basics', 2, None, '___sec23'),
|
||||
('Linear classifier', 2, None, '___sec24'),
|
||||
('Some selected properties', 2, None, '___sec25'),
|
||||
('The logistic function', 2, None, '___sec26'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'___sec27'),
|
||||
('Two parameters', 2, None, '___sec28'),
|
||||
('Maximum likelihood', 2, None, '___sec29'),
|
||||
('The cost function rewritten', 2, None, '___sec30'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec31'),
|
||||
('A more compact expression', 2, None, '___sec32'),
|
||||
('Extending to more predictors', 2, None, '___sec33'),
|
||||
('Including more classes', 2, None, '___sec34'),
|
||||
('More classes', 2, None, '___sec35'),
|
||||
('A simple classification problem', 2, None, '___sec36'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec37'),
|
||||
('Other measures in classification studies: Cancer Data again',
|
||||
2,
|
||||
None,
|
||||
'___sec38')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs005.html#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs023.html#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs024.html#___sec23" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs025.html#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs026.html#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs027.html#___sec26" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs028.html#___sec27" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs029.html#___sec28" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs030.html#___sec29" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs031.html#___sec30" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs032.html#___sec31" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs033.html#___sec32" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs034.html#___sec33" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec34" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs036.html#___sec35" style="font-size: 80%;">More classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs037.html#___sec36" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs038.html#___sec37" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs039.html#___sec38" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0035"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec34" class="anchor">Including more classes </h2>
|
||||
|
||||
<p>
|
||||
Till now we have mainly focused on two classes, the so-called binary
|
||||
system. Suppose we wish to extend to \( K \) classes. Let us for the sake
|
||||
of simplicity assume we have only two predictors. We have then
|
||||
following model
|
||||
|
||||
$$
|
||||
\log{\frac{p(C=1\vert x)}{p(K\vert x)}} = \beta_{10}+\beta_{11}x_1,
|
||||
$$
|
||||
|
||||
$$
|
||||
\log{\frac{p(C=2\vert x)}{p(K\vert x)}} = \beta_{20}+\beta_{21}x_1,
|
||||
$$
|
||||
|
||||
and so on till the class \( C=K-1 \) class
|
||||
$$
|
||||
\log{\frac{p(C=K-1\vert x)}{p(K\vert x)}} = \beta_{(K-1)0}+\beta_{(K-1)1}x_1,
|
||||
$$
|
||||
|
||||
<p>
|
||||
and the model is specified in term of \( K-1 \) so-called log-odds or
|
||||
<b>logit</b> transformations.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week38-bs034.html">«</a></li>
|
||||
<li><a href="._week38-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs027.html">28</a></li>
|
||||
<li><a href="._week38-bs028.html">29</a></li>
|
||||
<li><a href="._week38-bs029.html">30</a></li>
|
||||
<li><a href="._week38-bs030.html">31</a></li>
|
||||
<li><a href="._week38-bs031.html">32</a></li>
|
||||
<li><a href="._week38-bs032.html">33</a></li>
|
||||
<li><a href="._week38-bs033.html">34</a></li>
|
||||
<li><a href="._week38-bs034.html">35</a></li>
|
||||
<li class="active"><a href="._week38-bs035.html">36</a></li>
|
||||
<li><a href="._week38-bs036.html">37</a></li>
|
||||
<li><a href="._week38-bs037.html">38</a></li>
|
||||
<li><a href="._week38-bs038.html">39</a></li>
|
||||
<li><a href="._week38-bs039.html">40</a></li>
|
||||
<li><a href="._week38-bs036.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,283 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
||||
('Thursday September 17', 2, None, '___sec1'),
|
||||
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
||||
('Various steps in cross-validation', 2, None, '___sec3'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Cross-validation in brief', 2, None, '___sec5'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Cross-validation with Ridge', 2, None, '___sec9'),
|
||||
('The Ising model', 2, None, '___sec10'),
|
||||
('Reformulating the problem to suit regression',
|
||||
2,
|
||||
None,
|
||||
'___sec11'),
|
||||
('Linear regression', 2, None, '___sec12'),
|
||||
('Singular Value decomposition', 2, None, '___sec13'),
|
||||
('The one-dimensional Ising model', 2, None, '___sec14'),
|
||||
('Ridge regression', 2, None, '___sec15'),
|
||||
('LASSO regression', 2, None, '___sec16'),
|
||||
('Performance as function of the regularization parameter',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
||||
('Friday September 18: Intro to Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Logistic Regression', 2, None, '___sec20'),
|
||||
('Classification problems', 2, None, '___sec21'),
|
||||
('Optimization and Deep learning', 2, None, '___sec22'),
|
||||
('Basics', 2, None, '___sec23'),
|
||||
('Linear classifier', 2, None, '___sec24'),
|
||||
('Some selected properties', 2, None, '___sec25'),
|
||||
('The logistic function', 2, None, '___sec26'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'___sec27'),
|
||||
('Two parameters', 2, None, '___sec28'),
|
||||
('Maximum likelihood', 2, None, '___sec29'),
|
||||
('The cost function rewritten', 2, None, '___sec30'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec31'),
|
||||
('A more compact expression', 2, None, '___sec32'),
|
||||
('Extending to more predictors', 2, None, '___sec33'),
|
||||
('Including more classes', 2, None, '___sec34'),
|
||||
('More classes', 2, None, '___sec35'),
|
||||
('A simple classification problem', 2, None, '___sec36'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec37'),
|
||||
('Other measures in classification studies: Cancer Data again',
|
||||
2,
|
||||
None,
|
||||
'___sec38')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs005.html#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs023.html#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs024.html#___sec23" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs025.html#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs026.html#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs027.html#___sec26" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs028.html#___sec27" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs029.html#___sec28" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs030.html#___sec29" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs031.html#___sec30" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs032.html#___sec31" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs033.html#___sec32" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs034.html#___sec33" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs035.html#___sec34" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec35" style="font-size: 80%;">More classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs037.html#___sec36" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs038.html#___sec37" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs039.html#___sec38" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0036"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec35" class="anchor">More classes </h2>
|
||||
|
||||
<p>
|
||||
In our discussion of neural networks we will encounter the above again
|
||||
in terms of a slightly modified function, the so-called <b>Softmax</b> function.
|
||||
|
||||
<p>
|
||||
The softmax function is used in various multiclass classification
|
||||
methods, such as multinomial logistic regression (also known as
|
||||
softmax regression), multiclass linear discriminant analysis, naive
|
||||
Bayes classifiers, and artificial neural networks. Specifically, in
|
||||
multinomial logistic regression and linear discriminant analysis, the
|
||||
input to the function is the result of \( K \) distinct linear functions,
|
||||
and the predicted probability for the \( k \)-th class given a sample
|
||||
vector \( \hat{x} \) and a weighting vector \( \hat{\beta} \) is (with two
|
||||
predictors):
|
||||
|
||||
$$
|
||||
p(C=k\vert \mathbf {x} )=\frac{\exp{(\beta_{k0}+\beta_{k1}x_1)}}{1+\sum_{l=1}^{K-1}\exp{(\beta_{l0}+\beta_{l1}x_1)}}.
|
||||
$$
|
||||
|
||||
It is easy to extend to more predictors. The final class is
|
||||
$$
|
||||
p(C=K\vert \mathbf {x} )=\frac{1}{1+\sum_{l=1}^{K-1}\exp{(\beta_{l0}+\beta_{l1}x_1)}},
|
||||
$$
|
||||
|
||||
<p>
|
||||
and they sum to one. Our earlier discussions were all specialized to
|
||||
the case with two classes only. It is easy to see from the above that
|
||||
what we derived earlier is compatible with these equations.
|
||||
|
||||
<p>
|
||||
To find the optimal parameters we would typically use a gradient
|
||||
descent method. Newton's method and gradient descent methods are
|
||||
discussed in the material on <a href="https://compphysics.github.io/MachineLearning/doc/pub/Splines/html/Splines-bs.html" target="_self">optimization
|
||||
methods</a>.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week38-bs035.html">«</a></li>
|
||||
<li><a href="._week38-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs028.html">29</a></li>
|
||||
<li><a href="._week38-bs029.html">30</a></li>
|
||||
<li><a href="._week38-bs030.html">31</a></li>
|
||||
<li><a href="._week38-bs031.html">32</a></li>
|
||||
<li><a href="._week38-bs032.html">33</a></li>
|
||||
<li><a href="._week38-bs033.html">34</a></li>
|
||||
<li><a href="._week38-bs034.html">35</a></li>
|
||||
<li><a href="._week38-bs035.html">36</a></li>
|
||||
<li class="active"><a href="._week38-bs036.html">37</a></li>
|
||||
<li><a href="._week38-bs037.html">38</a></li>
|
||||
<li><a href="._week38-bs038.html">39</a></li>
|
||||
<li><a href="._week38-bs039.html">40</a></li>
|
||||
<li><a href="._week38-bs037.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
||||
('Thursday September 17', 2, None, '___sec1'),
|
||||
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
||||
('Various steps in cross-validation', 2, None, '___sec3'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Cross-validation in brief', 2, None, '___sec5'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Cross-validation with Ridge', 2, None, '___sec9'),
|
||||
('The Ising model', 2, None, '___sec10'),
|
||||
('Reformulating the problem to suit regression',
|
||||
2,
|
||||
None,
|
||||
'___sec11'),
|
||||
('Linear regression', 2, None, '___sec12'),
|
||||
('Singular Value decomposition', 2, None, '___sec13'),
|
||||
('The one-dimensional Ising model', 2, None, '___sec14'),
|
||||
('Ridge regression', 2, None, '___sec15'),
|
||||
('LASSO regression', 2, None, '___sec16'),
|
||||
('Performance as function of the regularization parameter',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
||||
('Friday September 18: Intro to Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Logistic Regression', 2, None, '___sec20'),
|
||||
('Classification problems', 2, None, '___sec21'),
|
||||
('Optimization and Deep learning', 2, None, '___sec22'),
|
||||
('Basics', 2, None, '___sec23'),
|
||||
('Linear classifier', 2, None, '___sec24'),
|
||||
('Some selected properties', 2, None, '___sec25'),
|
||||
('The logistic function', 2, None, '___sec26'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'___sec27'),
|
||||
('Two parameters', 2, None, '___sec28'),
|
||||
('Maximum likelihood', 2, None, '___sec29'),
|
||||
('The cost function rewritten', 2, None, '___sec30'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec31'),
|
||||
('A more compact expression', 2, None, '___sec32'),
|
||||
('Extending to more predictors', 2, None, '___sec33'),
|
||||
('Including more classes', 2, None, '___sec34'),
|
||||
('More classes', 2, None, '___sec35'),
|
||||
('A simple classification problem', 2, None, '___sec36'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec37'),
|
||||
('Other measures in classification studies: Cancer Data again',
|
||||
2,
|
||||
None,
|
||||
'___sec38')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs005.html#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs023.html#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs024.html#___sec23" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs025.html#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs026.html#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs027.html#___sec26" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs028.html#___sec27" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs029.html#___sec28" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs030.html#___sec29" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs031.html#___sec30" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs032.html#___sec31" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs033.html#___sec32" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs034.html#___sec33" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs035.html#___sec34" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs036.html#___sec35" style="font-size: 80%;">More classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec36" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs038.html#___sec37" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs039.html#___sec38" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0037"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec36" class="anchor">A simple classification problem </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets, linear_model
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">generate_data</span>():
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)
|
||||
X, y <span style="color: #666666">=</span> datasets<span style="color: #666666">.</span>make_moons(<span style="color: #666666">200</span>, noise<span style="color: #666666">=0.20</span>)
|
||||
<span style="color: #008000; font-weight: bold">return</span> X, y
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">visualize</span>(X, y, clf):
|
||||
plot_decision_boundary(<span style="color: #008000; font-weight: bold">lambda</span> x: clf<span style="color: #666666">.</span>predict(x), X, y)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_decision_boundary</span>(pred_func, X, y):
|
||||
<span style="color: #408080; font-style: italic"># Set min and max values and give it some padding</span>
|
||||
x_min, x_max <span style="color: #666666">=</span> X[:, <span style="color: #666666">0</span>]<span style="color: #666666">.</span>min() <span style="color: #666666">-</span> <span style="color: #666666">.5</span>, X[:, <span style="color: #666666">0</span>]<span style="color: #666666">.</span>max() <span style="color: #666666">+</span> <span style="color: #666666">.5</span>
|
||||
y_min, y_max <span style="color: #666666">=</span> X[:, <span style="color: #666666">1</span>]<span style="color: #666666">.</span>min() <span style="color: #666666">-</span> <span style="color: #666666">.5</span>, X[:, <span style="color: #666666">1</span>]<span style="color: #666666">.</span>max() <span style="color: #666666">+</span> <span style="color: #666666">.5</span>
|
||||
h <span style="color: #666666">=</span> <span style="color: #666666">0.01</span>
|
||||
<span style="color: #408080; font-style: italic"># Generate a grid of points with distance h between them</span>
|
||||
xx, yy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>meshgrid(np<span style="color: #666666">.</span>arange(x_min, x_max, h), np<span style="color: #666666">.</span>arange(y_min, y_max, h))
|
||||
<span style="color: #408080; font-style: italic"># Predict the function value for the whole gid</span>
|
||||
Z <span style="color: #666666">=</span> pred_func(np<span style="color: #666666">.</span>c_[xx<span style="color: #666666">.</span>ravel(), yy<span style="color: #666666">.</span>ravel()])
|
||||
Z <span style="color: #666666">=</span> Z<span style="color: #666666">.</span>reshape(xx<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic"># Plot the contour and training examples</span>
|
||||
plt<span style="color: #666666">.</span>contourf(xx, yy, Z, cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>Spectral)
|
||||
plt<span style="color: #666666">.</span>scatter(X[:, <span style="color: #666666">0</span>], X[:, <span style="color: #666666">1</span>], c<span style="color: #666666">=</span>y, cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>Spectral)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">classify</span>(X, y):
|
||||
clf <span style="color: #666666">=</span> linear_model<span style="color: #666666">.</span>LogisticRegressionCV()
|
||||
clf<span style="color: #666666">.</span>fit(X, y)
|
||||
<span style="color: #008000; font-weight: bold">return</span> clf
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">main</span>():
|
||||
X, y <span style="color: #666666">=</span> generate_data()
|
||||
<span style="color: #408080; font-style: italic"># visualize(X, y)</span>
|
||||
clf <span style="color: #666666">=</span> classify(X, y)
|
||||
visualize(X, y, clf)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #19177C">__name__</span> <span style="color: #666666">==</span> <span style="color: #BA2121">"__main__"</span>:
|
||||
main()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week38-bs036.html">«</a></li>
|
||||
<li><a href="._week38-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs029.html">30</a></li>
|
||||
<li><a href="._week38-bs030.html">31</a></li>
|
||||
<li><a href="._week38-bs031.html">32</a></li>
|
||||
<li><a href="._week38-bs032.html">33</a></li>
|
||||
<li><a href="._week38-bs033.html">34</a></li>
|
||||
<li><a href="._week38-bs034.html">35</a></li>
|
||||
<li><a href="._week38-bs035.html">36</a></li>
|
||||
<li><a href="._week38-bs036.html">37</a></li>
|
||||
<li class="active"><a href="._week38-bs037.html">38</a></li>
|
||||
<li><a href="._week38-bs038.html">39</a></li>
|
||||
<li><a href="._week38-bs039.html">40</a></li>
|
||||
<li><a href="._week38-bs038.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,274 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
||||
('Thursday September 17', 2, None, '___sec1'),
|
||||
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
||||
('Various steps in cross-validation', 2, None, '___sec3'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Cross-validation in brief', 2, None, '___sec5'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Cross-validation with Ridge', 2, None, '___sec9'),
|
||||
('The Ising model', 2, None, '___sec10'),
|
||||
('Reformulating the problem to suit regression',
|
||||
2,
|
||||
None,
|
||||
'___sec11'),
|
||||
('Linear regression', 2, None, '___sec12'),
|
||||
('Singular Value decomposition', 2, None, '___sec13'),
|
||||
('The one-dimensional Ising model', 2, None, '___sec14'),
|
||||
('Ridge regression', 2, None, '___sec15'),
|
||||
('LASSO regression', 2, None, '___sec16'),
|
||||
('Performance as function of the regularization parameter',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
||||
('Friday September 18: Intro to Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Logistic Regression', 2, None, '___sec20'),
|
||||
('Classification problems', 2, None, '___sec21'),
|
||||
('Optimization and Deep learning', 2, None, '___sec22'),
|
||||
('Basics', 2, None, '___sec23'),
|
||||
('Linear classifier', 2, None, '___sec24'),
|
||||
('Some selected properties', 2, None, '___sec25'),
|
||||
('The logistic function', 2, None, '___sec26'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'___sec27'),
|
||||
('Two parameters', 2, None, '___sec28'),
|
||||
('Maximum likelihood', 2, None, '___sec29'),
|
||||
('The cost function rewritten', 2, None, '___sec30'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec31'),
|
||||
('A more compact expression', 2, None, '___sec32'),
|
||||
('Extending to more predictors', 2, None, '___sec33'),
|
||||
('Including more classes', 2, None, '___sec34'),
|
||||
('More classes', 2, None, '___sec35'),
|
||||
('A simple classification problem', 2, None, '___sec36'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec37'),
|
||||
('Other measures in classification studies: Cancer Data again',
|
||||
2,
|
||||
None,
|
||||
'___sec38')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs005.html#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs023.html#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs024.html#___sec23" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs025.html#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs026.html#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs027.html#___sec26" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs028.html#___sec27" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs029.html#___sec28" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs030.html#___sec29" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs031.html#___sec30" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs032.html#___sec31" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs033.html#___sec32" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs034.html#___sec33" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs035.html#___sec34" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs036.html#___sec35" style="font-size: 80%;">More classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs037.html#___sec36" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec37" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs039.html#___sec38" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0038"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec37" class="anchor">Cancer Data again now with Decision Trees and other Methods </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Load the data</span>
|
||||
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
||||
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(X_train<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(X_test<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
||||
logreg <span style="color: #666666">=</span> LogisticRegression(solver<span style="color: #666666">=</span><span style="color: #BA2121">'lbfgs'</span>)
|
||||
logreg<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test set accuracy with Logistic Regression: {:.2f}"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test,y_test)))
|
||||
<span style="color: #408080; font-style: italic">#now scale the data</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
scaler<span style="color: #666666">.</span>fit(X_train)
|
||||
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
||||
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
||||
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
||||
logreg<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test set accuracy Logistic Regression with scaled data: {:.2f}"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week38-bs037.html">«</a></li>
|
||||
<li><a href="._week38-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs030.html">31</a></li>
|
||||
<li><a href="._week38-bs031.html">32</a></li>
|
||||
<li><a href="._week38-bs032.html">33</a></li>
|
||||
<li><a href="._week38-bs033.html">34</a></li>
|
||||
<li><a href="._week38-bs034.html">35</a></li>
|
||||
<li><a href="._week38-bs035.html">36</a></li>
|
||||
<li><a href="._week38-bs036.html">37</a></li>
|
||||
<li><a href="._week38-bs037.html">38</a></li>
|
||||
<li class="active"><a href="._week38-bs038.html">39</a></li>
|
||||
<li><a href="._week38-bs039.html">40</a></li>
|
||||
<li><a href="._week38-bs039.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,292 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 38', 2, None, '___sec0'),
|
||||
('Thursday September 17', 2, None, '___sec1'),
|
||||
('Ridge and LASSO Regression, reminder', 2, None, '___sec2'),
|
||||
('Various steps in cross-validation', 2, None, '___sec3'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Cross-validation in brief', 2, None, '___sec5'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'___sec6'),
|
||||
('Bias-Variance tradeoff with Bootstrap', 2, None, '___sec7'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'___sec8'),
|
||||
('Cross-validation with Ridge', 2, None, '___sec9'),
|
||||
('The Ising model', 2, None, '___sec10'),
|
||||
('Reformulating the problem to suit regression',
|
||||
2,
|
||||
None,
|
||||
'___sec11'),
|
||||
('Linear regression', 2, None, '___sec12'),
|
||||
('Singular Value decomposition', 2, None, '___sec13'),
|
||||
('The one-dimensional Ising model', 2, None, '___sec14'),
|
||||
('Ridge regression', 2, None, '___sec15'),
|
||||
('LASSO regression', 2, None, '___sec16'),
|
||||
('Performance as function of the regularization parameter',
|
||||
2,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Finding the optimal value of $\\lambda$', 2, None, '___sec18'),
|
||||
('Friday September 18: Intro to Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec19'),
|
||||
('Logistic Regression', 2, None, '___sec20'),
|
||||
('Classification problems', 2, None, '___sec21'),
|
||||
('Optimization and Deep learning', 2, None, '___sec22'),
|
||||
('Basics', 2, None, '___sec23'),
|
||||
('Linear classifier', 2, None, '___sec24'),
|
||||
('Some selected properties', 2, None, '___sec25'),
|
||||
('The logistic function', 2, None, '___sec26'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'___sec27'),
|
||||
('Two parameters', 2, None, '___sec28'),
|
||||
('Maximum likelihood', 2, None, '___sec29'),
|
||||
('The cost function rewritten', 2, None, '___sec30'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec31'),
|
||||
('A more compact expression', 2, None, '___sec32'),
|
||||
('Extending to more predictors', 2, None, '___sec33'),
|
||||
('Including more classes', 2, None, '___sec34'),
|
||||
('More classes', 2, None, '___sec35'),
|
||||
('A simple classification problem', 2, None, '___sec36'),
|
||||
('Cancer Data again now with Decision Trees and other Methods',
|
||||
2,
|
||||
None,
|
||||
'___sec37'),
|
||||
('Other measures in classification studies: Cancer Data again',
|
||||
2,
|
||||
None,
|
||||
'___sec38')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs001.html#___sec0" style="font-size: 80%;">Plans for week 38</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs002.html#___sec1" style="font-size: 80%;">Thursday September 17</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs003.html#___sec2" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs004.html#___sec3" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs005.html#___sec4" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs006.html#___sec5" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs007.html#___sec6" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs008.html#___sec7" style="font-size: 80%;">Bias-Variance tradeoff with Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs009.html#___sec8" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs010.html#___sec9" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs011.html#___sec10" style="font-size: 80%;">The Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs012.html#___sec11" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs013.html#___sec12" style="font-size: 80%;">Linear regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs014.html#___sec13" style="font-size: 80%;">Singular Value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs015.html#___sec14" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs016.html#___sec15" style="font-size: 80%;">Ridge regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs017.html#___sec16" style="font-size: 80%;">LASSO regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs018.html#___sec17" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs019.html#___sec18" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs020.html#___sec19" style="font-size: 80%;">Friday September 18: Intro to Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs021.html#___sec20" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs022.html#___sec21" style="font-size: 80%;">Classification problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs023.html#___sec22" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs024.html#___sec23" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs025.html#___sec24" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs026.html#___sec25" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs027.html#___sec26" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs028.html#___sec27" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs029.html#___sec28" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs030.html#___sec29" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs031.html#___sec30" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs032.html#___sec31" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs033.html#___sec32" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs034.html#___sec33" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs035.html#___sec34" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs036.html#___sec35" style="font-size: 80%;">More classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs037.html#___sec36" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week38-bs038.html#___sec37" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec38" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0039"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec38" class="anchor">Other measures in classification studies: Cancer Data again </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Load the data</span>
|
||||
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
||||
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(X_train<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(X_test<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
||||
logreg <span style="color: #666666">=</span> LogisticRegression(solver<span style="color: #666666">=</span><span style="color: #BA2121">'lbfgs'</span>)
|
||||
logreg<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test set accuracy with Logistic Regression: {:.2f}"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test,y_test)))
|
||||
<span style="color: #408080; font-style: italic">#now scale the data</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
scaler<span style="color: #666666">.</span>fit(X_train)
|
||||
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
||||
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
||||
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
||||
logreg<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test set accuracy Logistic Regression with scaled data: {:.2f}"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> LabelEncoder
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
|
||||
<span style="color: #408080; font-style: italic">#Cross validation</span>
|
||||
accuracy <span style="color: #666666">=</span> cross_validate(logreg,X_test_scaled,y_test,cv<span style="color: #666666">=10</span>)[<span style="color: #BA2121">'test_score'</span>]
|
||||
<span style="color: #008000; font-weight: bold">print</span>(accuracy)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Test set accuracy with Logistic Regression and scaled data: {:.2f}"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
y_pred <span style="color: #666666">=</span> logreg<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000">True</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
y_probas <span style="color: #666666">=</span> logreg<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week38-bs038.html">«</a></li>
|
||||
<li><a href="._week38-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week38-bs031.html">32</a></li>
|
||||
<li><a href="._week38-bs032.html">33</a></li>
|
||||
<li><a href="._week38-bs033.html">34</a></li>
|
||||
<li><a href="._week38-bs034.html">35</a></li>
|
||||
<li><a href="._week38-bs035.html">36</a></li>
|
||||
<li><a href="._week38-bs036.html">37</a></li>
|
||||
<li><a href="._week38-bs037.html">38</a></li>
|
||||
<li><a href="._week38-bs038.html">39</a></li>
|
||||
<li class="active"><a href="._week38-bs039.html">40</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user