Morten Hjorth-Jensen
496 lines
33 KiB
HTML
496 lines
33 KiB
HTML
<!--
|
|
Automatically generated HTML file from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
|
|
|
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
|
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
|
|
<style type="text/css">
|
|
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
|
|
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plans for week 38', 2, None, 'plans-for-week-38'),
|
|
('Thursday September 23', 2, None, 'thursday-september-23'),
|
|
('Ridge and LASSO Regression, reminder',
|
|
2,
|
|
None,
|
|
'ridge-and-lasso-regression-reminder'),
|
|
('Various steps in cross-validation',
|
|
2,
|
|
None,
|
|
'various-steps-in-cross-validation'),
|
|
('How to set up the cross-validation for Ridge and/or Lasso',
|
|
2,
|
|
None,
|
|
'how-to-set-up-the-cross-validation-for-ridge-and-or-lasso'),
|
|
('Cross-validation in brief',
|
|
2,
|
|
None,
|
|
'cross-validation-in-brief'),
|
|
('Code Example for Cross-validation and $k$-fold '
|
|
'Cross-validation',
|
|
2,
|
|
None,
|
|
'code-example-for-cross-validation-and-k-fold-cross-validation'),
|
|
('To think about, first part',
|
|
2,
|
|
None,
|
|
'to-think-about-first-part'),
|
|
('More thinking', 2, None, 'more-thinking'),
|
|
('Still thinking', 2, None, 'still-thinking'),
|
|
('What does centering (subtracting the mean values) mean '
|
|
'mathematically?',
|
|
2,
|
|
None,
|
|
'what-does-centering-subtracting-the-mean-values-mean-mathematically'),
|
|
('Further Manipulations', 2, None, 'further-manipulations'),
|
|
('Wrapping it up', 2, None, 'wrapping-it-up'),
|
|
('Linear Regression code, Intercept handling first',
|
|
2,
|
|
None,
|
|
'linear-regression-code-intercept-handling-first'),
|
|
('Code Examples', 2, None, 'code-examples'),
|
|
('Taking out the mean', 2, None, 'taking-out-the-mean'),
|
|
('More complicated Example: The Ising model',
|
|
2,
|
|
None,
|
|
'more-complicated-example-the-ising-model'),
|
|
('Reformulating the problem to suit regression',
|
|
2,
|
|
None,
|
|
'reformulating-the-problem-to-suit-regression'),
|
|
('Linear regression', 2, None, 'linear-regression'),
|
|
('Singular Value decomposition',
|
|
2,
|
|
None,
|
|
'singular-value-decomposition'),
|
|
('The one-dimensional Ising model',
|
|
2,
|
|
None,
|
|
'the-one-dimensional-ising-model'),
|
|
('Ridge regression', 2, None, 'ridge-regression'),
|
|
('LASSO regression', 2, None, 'lasso-regression'),
|
|
('Performance as function of the regularization parameter',
|
|
2,
|
|
None,
|
|
'performance-as-function-of-the-regularization-parameter'),
|
|
('Finding the optimal value of $\\lambda$',
|
|
2,
|
|
None,
|
|
'finding-the-optimal-value-of-lambda'),
|
|
('Logistic Regression', 2, None, 'logistic-regression'),
|
|
('Classification problems', 2, None, 'classification-problems'),
|
|
('Optimization and Deep learning',
|
|
2,
|
|
None,
|
|
'optimization-and-deep-learning'),
|
|
('Basics', 2, None, 'basics'),
|
|
('Linear classifier', 2, None, 'linear-classifier'),
|
|
('Some selected properties', 2, None, 'some-selected-properties'),
|
|
('Simple example', 2, None, 'simple-example'),
|
|
('Plotting the mean value for each group',
|
|
2,
|
|
None,
|
|
'plotting-the-mean-value-for-each-group'),
|
|
('The logistic function', 2, None, 'the-logistic-function'),
|
|
('Examples of likelihood functions used in logistic regression '
|
|
'and nueral networks',
|
|
2,
|
|
None,
|
|
'examples-of-likelihood-functions-used-in-logistic-regression-and-nueral-networks'),
|
|
('Two parameters', 2, None, 'two-parameters'),
|
|
('Maximum likelihood', 2, None, 'maximum-likelihood'),
|
|
('The cost function rewritten',
|
|
2,
|
|
None,
|
|
'the-cost-function-rewritten'),
|
|
('Minimizing the cross entropy',
|
|
2,
|
|
None,
|
|
'minimizing-the-cross-entropy'),
|
|
('A more compact expression',
|
|
2,
|
|
None,
|
|
'a-more-compact-expression'),
|
|
('Extending to more predictors',
|
|
2,
|
|
None,
|
|
'extending-to-more-predictors'),
|
|
('Including more classes', 2, None, 'including-more-classes'),
|
|
('More classes', 2, None, 'more-classes'),
|
|
('Friday September 24', 2, None, 'friday-september-24'),
|
|
('Wisconsin Cancer Data', 2, None, 'wisconsin-cancer-data'),
|
|
('Using the correlation matrix',
|
|
2,
|
|
None,
|
|
'using-the-correlation-matrix'),
|
|
('Discussing the correlation data',
|
|
2,
|
|
None,
|
|
'discussing-the-correlation-data'),
|
|
('Other measures in classification studies: Cancer Data again',
|
|
2,
|
|
None,
|
|
'other-measures-in-classification-studies-cancer-data-again'),
|
|
('Friday September 25', 2, None, 'friday-september-25'),
|
|
('Optimization, the central part of any Machine Learning '
|
|
'algortithm',
|
|
2,
|
|
None,
|
|
'optimization-the-central-part-of-any-machine-learning-algortithm'),
|
|
('Revisiting our Logistic Regression case',
|
|
2,
|
|
None,
|
|
'revisiting-our-logistic-regression-case'),
|
|
('The equations to solve', 2, None, 'the-equations-to-solve'),
|
|
("Solving using Newton-Raphson's method",
|
|
2,
|
|
None,
|
|
'solving-using-newton-raphson-s-method'),
|
|
("Brief reminder on Newton-Raphson's method",
|
|
2,
|
|
None,
|
|
'brief-reminder-on-newton-raphson-s-method'),
|
|
('The equations', 2, None, 'the-equations'),
|
|
('Simple geometric interpretation',
|
|
2,
|
|
None,
|
|
'simple-geometric-interpretation'),
|
|
('Extending to more than one variable',
|
|
2,
|
|
None,
|
|
'extending-to-more-than-one-variable'),
|
|
('Steepest descent', 2, None, 'steepest-descent'),
|
|
('More on Steepest descent', 2, None, 'more-on-steepest-descent'),
|
|
('The ideal', 2, None, 'the-ideal'),
|
|
('The sensitiveness of the gradient descent',
|
|
2,
|
|
None,
|
|
'the-sensitiveness-of-the-gradient-descent')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week38-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
|
</div>
|
|
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week38-bs001.html#plans-for-week-38" style="font-size: 80%;">Plans for week 38</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs002.html#thursday-september-23" style="font-size: 80%;">Thursday September 23</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs003.html#ridge-and-lasso-regression-reminder" style="font-size: 80%;">Ridge and LASSO Regression, reminder</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs004.html#various-steps-in-cross-validation" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs005.html#how-to-set-up-the-cross-validation-for-ridge-and-or-lasso" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs006.html#cross-validation-in-brief" style="font-size: 80%;">Cross-validation in brief</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs007.html#code-example-for-cross-validation-and-k-fold-cross-validation" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs008.html#to-think-about-first-part" style="font-size: 80%;">To think about, first part</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs009.html#more-thinking" style="font-size: 80%;">More thinking</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs010.html#still-thinking" style="font-size: 80%;">Still thinking</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs011.html#what-does-centering-subtracting-the-mean-values-mean-mathematically" style="font-size: 80%;">What does centering (subtracting the mean values) mean mathematically?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs012.html#further-manipulations" style="font-size: 80%;">Further Manipulations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs013.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
|
|
<!-- navigation toc: --> <li><a href="#linear-regression-code-intercept-handling-first" style="font-size: 80%;">Linear Regression code, Intercept handling first</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs015.html#code-examples" style="font-size: 80%;">Code Examples</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs016.html#taking-out-the-mean" style="font-size: 80%;">Taking out the mean</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs017.html#more-complicated-example-the-ising-model" style="font-size: 80%;">More complicated Example: The Ising model</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs018.html#reformulating-the-problem-to-suit-regression" style="font-size: 80%;">Reformulating the problem to suit regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs019.html#linear-regression" style="font-size: 80%;">Linear regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs020.html#singular-value-decomposition" style="font-size: 80%;">Singular Value decomposition</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs021.html#the-one-dimensional-ising-model" style="font-size: 80%;">The one-dimensional Ising model</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs022.html#ridge-regression" style="font-size: 80%;">Ridge regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs023.html#lasso-regression" style="font-size: 80%;">LASSO regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs024.html#performance-as-function-of-the-regularization-parameter" style="font-size: 80%;">Performance as function of the regularization parameter</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs025.html#finding-the-optimal-value-of-lambda" style="font-size: 80%;">Finding the optimal value of \( \lambda \)</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs026.html#logistic-regression" style="font-size: 80%;">Logistic Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs027.html#classification-problems" style="font-size: 80%;">Classification problems</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs028.html#optimization-and-deep-learning" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs029.html#basics" style="font-size: 80%;">Basics</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs030.html#linear-classifier" style="font-size: 80%;">Linear classifier</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs031.html#some-selected-properties" style="font-size: 80%;">Some selected properties</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs032.html#simple-example" style="font-size: 80%;">Simple example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs033.html#plotting-the-mean-value-for-each-group" style="font-size: 80%;">Plotting the mean value for each group</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs034.html#the-logistic-function" style="font-size: 80%;">The logistic function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs035.html#examples-of-likelihood-functions-used-in-logistic-regression-and-nueral-networks" style="font-size: 80%;">Examples of likelihood functions used in logistic regression and nueral networks</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs036.html#two-parameters" style="font-size: 80%;">Two parameters</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs037.html#maximum-likelihood" style="font-size: 80%;">Maximum likelihood</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs038.html#the-cost-function-rewritten" style="font-size: 80%;">The cost function rewritten</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs039.html#minimizing-the-cross-entropy" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs040.html#a-more-compact-expression" style="font-size: 80%;">A more compact expression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs041.html#extending-to-more-predictors" style="font-size: 80%;">Extending to more predictors</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs042.html#including-more-classes" style="font-size: 80%;">Including more classes</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs043.html#more-classes" style="font-size: 80%;">More classes</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs044.html#friday-september-24" style="font-size: 80%;">Friday September 24</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs045.html#wisconsin-cancer-data" style="font-size: 80%;">Wisconsin Cancer Data</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs046.html#using-the-correlation-matrix" style="font-size: 80%;">Using the correlation matrix</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs047.html#discussing-the-correlation-data" style="font-size: 80%;">Discussing the correlation data</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs048.html#other-measures-in-classification-studies-cancer-data-again" style="font-size: 80%;">Other measures in classification studies: Cancer Data again</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs049.html#friday-september-25" style="font-size: 80%;">Friday September 25</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs050.html#optimization-the-central-part-of-any-machine-learning-algortithm" style="font-size: 80%;">Optimization, the central part of any Machine Learning algortithm</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs051.html#revisiting-our-logistic-regression-case" style="font-size: 80%;">Revisiting our Logistic Regression case</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs052.html#the-equations-to-solve" style="font-size: 80%;">The equations to solve</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs053.html#solving-using-newton-raphson-s-method" style="font-size: 80%;">Solving using Newton-Raphson's method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs054.html#brief-reminder-on-newton-raphson-s-method" style="font-size: 80%;">Brief reminder on Newton-Raphson's method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs055.html#the-equations" style="font-size: 80%;">The equations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs056.html#simple-geometric-interpretation" style="font-size: 80%;">Simple geometric interpretation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs057.html#extending-to-more-than-one-variable" style="font-size: 80%;">Extending to more than one variable</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs058.html#steepest-descent" style="font-size: 80%;">Steepest descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs059.html#more-on-steepest-descent" style="font-size: 80%;">More on Steepest descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs060.html#the-ideal" style="font-size: 80%;">The ideal</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week38-bs061.html#the-sensitiveness-of-the-gradient-descent" style="font-size: 80%;">The sensitiveness of the gradient descent</a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
|
|
<div class="container">
|
|
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
|
|
<a name="part0014"></a>
|
|
<!-- !split -->
|
|
|
|
<h2 id="linear-regression-code-intercept-handling-first" class="anchor">Linear Regression code, Intercept handling first </h2>
|
|
|
|
<p>
|
|
This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (<em>code example thanks to Øyvind Sigmundson Schøyen</em>). Here our scaling of the data is done by subtracting the mean values only.
|
|
Note also that we do not split the data into training and test.
|
|
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
|
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LinearRegression
|
|
|
|
|
|
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">2021</span>)
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">MSE</span>(y_data,y_model):
|
|
n <span style="color: #666666">=</span> np<span style="color: #666666">.</span>size(y_model)
|
|
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>sum((y_data<span style="color: #666666">-</span>y_model)<span style="color: #666666">**2</span>)<span style="color: #666666">/</span>n
|
|
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">fit_beta</span>(X, y):
|
|
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>pinv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X) <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y
|
|
|
|
|
|
true_beta <span style="color: #666666">=</span> [<span style="color: #666666">2</span>, <span style="color: #666666">0.5</span>, <span style="color: #666666">3.7</span>]
|
|
|
|
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">11</span>)
|
|
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(
|
|
np<span style="color: #666666">.</span>asarray([x <span style="color: #666666">**</span> p <span style="color: #666666">*</span> b <span style="color: #008000; font-weight: bold">for</span> p, b <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(true_beta)]), axis<span style="color: #666666">=0</span>
|
|
) <span style="color: #666666">+</span> <span style="color: #666666">0.1</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal(size<span style="color: #666666">=</span><span style="color: #008000">len</span>(x))
|
|
|
|
degree <span style="color: #666666">=</span> <span style="color: #666666">3</span>
|
|
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(x), degree))
|
|
|
|
<span style="color: #408080; font-style: italic"># Include the intercept in the design matrix</span>
|
|
<span style="color: #008000; font-weight: bold">for</span> p <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(degree):
|
|
X[:, p] <span style="color: #666666">=</span> x <span style="color: #666666">**</span> p
|
|
|
|
beta <span style="color: #666666">=</span> fit_beta(X, y)
|
|
|
|
<span style="color: #408080; font-style: italic"># Intercept is included in the design matrix</span>
|
|
skl <span style="color: #666666">=</span> LinearRegression(fit_intercept<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>)<span style="color: #666666">.</span>fit(X, y)
|
|
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"True beta: </span><span style="color: #BB6688; font-weight: bold">{</span>true_beta<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"Fitted beta: </span><span style="color: #BB6688; font-weight: bold">{</span>beta<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"Sklearn fitted beta: </span><span style="color: #BB6688; font-weight: bold">{</span>skl<span style="color: #666666">.</span>coef_<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
ypredictOwn <span style="color: #666666">=</span> X <span style="color: #666666">@</span> beta
|
|
ypredictSKL <span style="color: #666666">=</span> skl<span style="color: #666666">.</span>predict(X)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"MSE with intercept column"</span>)
|
|
<span style="color: #008000">print</span>(MSE(y,ypredictOwn))
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"MSE with intercept column from SKL"</span>)
|
|
<span style="color: #008000">print</span>(MSE(y,ypredictSKL))
|
|
|
|
|
|
plt<span style="color: #666666">.</span>figure()
|
|
plt<span style="color: #666666">.</span>scatter(x, y, label<span style="color: #666666">=</span><span style="color: #BA2121">"Data"</span>)
|
|
plt<span style="color: #666666">.</span>plot(x, X <span style="color: #666666">@</span> beta, label<span style="color: #666666">=</span><span style="color: #BA2121">"Fit"</span>)
|
|
plt<span style="color: #666666">.</span>plot(x, skl<span style="color: #666666">.</span>predict(X), label<span style="color: #666666">=</span><span style="color: #BA2121">"Sklearn (fit_intercept=False)"</span>)
|
|
|
|
|
|
<span style="color: #408080; font-style: italic"># Do not include the intercept in the design matrix</span>
|
|
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(x), degree <span style="color: #666666">-</span> <span style="color: #666666">1</span>))
|
|
|
|
<span style="color: #008000; font-weight: bold">for</span> p <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(degree <span style="color: #666666">-</span> <span style="color: #666666">1</span>):
|
|
X[:, p] <span style="color: #666666">=</span> x <span style="color: #666666">**</span> (p <span style="color: #666666">+</span> <span style="color: #666666">1</span>)
|
|
|
|
<span style="color: #408080; font-style: italic"># Intercept is not included in the design matrix</span>
|
|
skl <span style="color: #666666">=</span> LinearRegression(fit_intercept<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)<span style="color: #666666">.</span>fit(X, y)
|
|
|
|
<span style="color: #408080; font-style: italic"># Use centered values for X and y when computing coefficients</span>
|
|
y_offset <span style="color: #666666">=</span> np<span style="color: #666666">.</span>average(y, axis<span style="color: #666666">=0</span>)
|
|
X_offset <span style="color: #666666">=</span> np<span style="color: #666666">.</span>average(X, axis<span style="color: #666666">=0</span>)
|
|
|
|
beta <span style="color: #666666">=</span> fit_beta(X <span style="color: #666666">-</span> X_offset, y <span style="color: #666666">-</span> y_offset)
|
|
intercept <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean(y_offset <span style="color: #666666">-</span> X_offset <span style="color: #666666">@</span> beta)
|
|
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"Manual intercept: </span><span style="color: #BB6688; font-weight: bold">{</span>intercept<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"Fitted beta (wiothout intercept): </span><span style="color: #BB6688; font-weight: bold">{</span>beta<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"Sklearn intercept: </span><span style="color: #BB6688; font-weight: bold">{</span>skl<span style="color: #666666">.</span>intercept_<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"Sklearn fitted beta (without intercept): </span><span style="color: #BB6688; font-weight: bold">{</span>skl<span style="color: #666666">.</span>coef_<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
ypredictOwn <span style="color: #666666">=</span> X <span style="color: #666666">@</span> beta
|
|
ypredictSKL <span style="color: #666666">=</span> skl<span style="color: #666666">.</span>predict(X)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"MSE with Manual intercept"</span>)
|
|
<span style="color: #008000">print</span>(MSE(y,ypredictOwn<span style="color: #666666">+</span>intercept))
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"MSE with Sklearn intercept"</span>)
|
|
<span style="color: #008000">print</span>(MSE(y,ypredictSKL))
|
|
|
|
plt<span style="color: #666666">.</span>plot(x, X <span style="color: #666666">@</span> beta <span style="color: #666666">+</span> intercept, <span style="color: #BA2121">"--"</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Fit (manual intercept)"</span>)
|
|
plt<span style="color: #666666">.</span>plot(x, skl<span style="color: #666666">.</span>predict(X), <span style="color: #BA2121">"--"</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Sklearn (fit_intercept=True)"</span>)
|
|
plt<span style="color: #666666">.</span>grid()
|
|
plt<span style="color: #666666">.</span>legend()
|
|
|
|
plt<span style="color: #666666">.</span>show()
|
|
</pre></div>
|
|
<p>
|
|
The intercept is the value of our output/target variable
|
|
when all our features are zero and our function crosses the \( y \)-axis (for a one-dimensional case).
|
|
|
|
<p>
|
|
Printing the MSE, we see first that both methods give the same MSE, as
|
|
they should. However, when we move to for example Ridge regression,
|
|
the way we treat the intercept may give a larger or smaller MSE,
|
|
meaning that the MSE can be penalized by the value of the
|
|
intercept. Not including the intercept in the fit, means that the
|
|
regularization term does not include \( \beta_0 \). For different values
|
|
of \( \lambda \), this may lead to differeing MSE values.
|
|
|
|
<p>
|
|
To remind the reader, the regularization term, with the intercept in Ridge regression is given by
|
|
$$
|
|
\lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=0}^{p-1}\beta_j^2,
|
|
$$
|
|
|
|
but when we take out the intercept, this equation becomes
|
|
$$
|
|
\lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=1}^{p-1}\beta_j^2.
|
|
$$
|
|
|
|
<p>
|
|
For Lasso regression we have
|
|
$$
|
|
\lambda \vert\vert \boldsymbol{\beta} \vert\vert_1 = \lambda \sum_{j=1}^{p-1}\vert\beta_j\vert.
|
|
$$
|
|
|
|
<p>
|
|
It means that, when scaling the design matrix and the outputs/targets, by subtracting the mean values, we have an optimization problem which is not penalized by the intercept. The MSE value can then be smaller since it focuses only on the remaining quantities. If we however bring back the intercept, we will get a MSE which then contains the intercept.
|
|
|
|
<p>
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week38-bs013.html">«</a></li>
|
|
<li><a href="._week38-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week38-bs006.html">7</a></li>
|
|
<li><a href="._week38-bs007.html">8</a></li>
|
|
<li><a href="._week38-bs008.html">9</a></li>
|
|
<li><a href="._week38-bs009.html">10</a></li>
|
|
<li><a href="._week38-bs010.html">11</a></li>
|
|
<li><a href="._week38-bs011.html">12</a></li>
|
|
<li><a href="._week38-bs012.html">13</a></li>
|
|
<li><a href="._week38-bs013.html">14</a></li>
|
|
<li class="active"><a href="._week38-bs014.html">15</a></li>
|
|
<li><a href="._week38-bs015.html">16</a></li>
|
|
<li><a href="._week38-bs016.html">17</a></li>
|
|
<li><a href="._week38-bs017.html">18</a></li>
|
|
<li><a href="._week38-bs018.html">19</a></li>
|
|
<li><a href="._week38-bs019.html">20</a></li>
|
|
<li><a href="._week38-bs020.html">21</a></li>
|
|
<li><a href="._week38-bs021.html">22</a></li>
|
|
<li><a href="._week38-bs022.html">23</a></li>
|
|
<li><a href="._week38-bs023.html">24</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week38-bs061.html">62</a></li>
|
|
<li><a href="._week38-bs015.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
|
</footer>
|
|
-->
|
|
|
|
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
|
|
|
|
</body>
|
|
</html>
|
|
|
|
|