Files
FYS-STK4155/doc/pub/week36/html/._week36-bs030.html
T
2023-05-29 22:09:43 +02:00

373 lines
22 KiB
HTML

<!--
HTML file automatically generated from DocOnce source
(https://github.com/doconce/doconce/)
doconce format html week36.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week36-bs --no_mako
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="description" content="Week 36: Statistical interpretation of Linear Regression and Resampling techniques">
<title>Week 36: Statistical interpretation of Linear Regression and Resampling techniques</title>
<!-- Bootstrap style: bootstrap -->
<!-- doconce format html week36.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week36-bs --no_mako -->
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
<!-- not necessary
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
-->
<style type="text/css">
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
.dropdown-menu {
height: auto;
max-height: 400px;
overflow-x: hidden;
}
/* Adds an invisible element before each target to offset for the navigation
bar */
.anchor::before {
content:"";
display:block;
height:50px; /* fixed header height for style bootstrap */
margin:-50px 0 0; /* negative fixed header height */
}
</style>
</head>
<!-- tocinfo
{'highest level': 2,
'sections': [('Plans for week 36', 2, None, 'plans-for-week-36'),
('Summary from last Week and discussion of SVD, Ridge and Lasso '
'regression with examples',
2,
None,
'summary-from-last-week-and-discussion-of-svd-ridge-and-lasso-regression-with-examples'),
('Linear Regression and the SVD',
2,
None,
'linear-regression-and-the-svd'),
('What does it mean?', 2, None, 'what-does-it-mean'),
('And finally $\\boldsymbol{X}\\boldsymbol{X}^T$',
2,
None,
'and-finally-boldsymbol-x-boldsymbol-x-t'),
('Code for SVD and Inversion of Matrices',
2,
None,
'code-for-svd-and-inversion-of-matrices'),
('Inverse of Rectangular Matrix',
2,
None,
'inverse-of-rectangular-matrix'),
('Ridge and LASSO Regression',
2,
None,
'ridge-and-lasso-regression'),
('From OLS to Ridge and Lasso',
2,
None,
'from-ols-to-ridge-and-lasso'),
('Deriving the Ridge Regression Equations',
2,
None,
'deriving-the-ridge-regression-equations'),
('Note on Scikit-Learn', 2, None, 'note-on-scikit-learn'),
('Comparison with OLS', 2, None, 'comparison-with-ols'),
('SVD analysis', 2, None, 'svd-analysis'),
('Interpreting the Ridge results',
2,
None,
'interpreting-the-ridge-results'),
('More interpretations', 2, None, 'more-interpretations'),
('Deriving the Lasso Regression Equations',
2,
None,
'deriving-the-lasso-regression-equations'),
('Simple example to illustrate Ordinary Least Squares, Ridge and '
'Lasso Regression',
2,
None,
'simple-example-to-illustrate-ordinary-least-squares-ridge-and-lasso-regression'),
('Ridge Regression', 2, None, 'ridge-regression'),
('Lasso Regression', 2, None, 'lasso-regression'),
('Yet another Example', 2, None, 'yet-another-example'),
('The OLS case', 2, None, 'the-ols-case'),
('The Ridge case', 2, None, 'the-ridge-case'),
('Writing the Cost Function',
2,
None,
'writing-the-cost-function'),
('Lasso case', 2, None, 'lasso-case'),
('The first Case', 2, None, 'the-first-case'),
('Simple code for solving the above problem',
2,
None,
'simple-code-for-solving-the-above-problem'),
('With Lasso Regression', 2, None, 'with-lasso-regression'),
('Another Example, now with a polynomial fit',
2,
None,
'another-example-now-with-a-polynomial-fit'),
('To think about, first part',
2,
None,
'to-think-about-first-part'),
('More thinking', 2, None, 'more-thinking'),
('Still thinking', 2, None, 'still-thinking'),
('What does centering (subtracting the mean values) mean '
'mathematically?',
2,
None,
'what-does-centering-subtracting-the-mean-values-mean-mathematically'),
('Further Manipulations', 2, None, 'further-manipulations'),
('Wrapping it up', 2, None, 'wrapping-it-up'),
('Linear Regression code, Intercept handling first',
2,
None,
'linear-regression-code-intercept-handling-first'),
('Code Examples', 2, None, 'code-examples'),
('Taking out the mean', 2, None, 'taking-out-the-mean'),
('Friday September 9', 2, None, 'friday-september-9'),
('Linking the regression analysis with a statistical '
'interpretation',
2,
None,
'linking-the-regression-analysis-with-a-statistical-interpretation'),
('Assumptions made', 2, None, 'assumptions-made'),
('Expectation value and variance',
2,
None,
'expectation-value-and-variance'),
('Expectation value and variance for $\\boldsymbol{\\beta}$',
2,
None,
'expectation-value-and-variance-for-boldsymbol-beta'),
('Deriving OLS from a probability distribution',
2,
None,
'deriving-ols-from-a-probability-distribution'),
('Independent and Identically Distrubuted (iid)',
2,
None,
'independent-and-identically-distrubuted-iid'),
('Maximum Likelihood Estimation (MLE)',
2,
None,
'maximum-likelihood-estimation-mle'),
('A new Cost Function', 2, None, 'a-new-cost-function'),
("More basic Statistics and Bayes' theorem",
2,
None,
'more-basic-statistics-and-bayes-theorem'),
('Marginal Probability', 2, None, 'marginal-probability'),
('Conditional Probability', 2, None, 'conditional-probability'),
("Bayes' Theorem", 2, None, 'bayes-theorem'),
("Interpretations of Bayes' Theorem",
2,
None,
'interpretations-of-bayes-theorem'),
("Example of Usage of Bayes' theorem",
2,
None,
'example-of-usage-of-bayes-theorem'),
('Doing it correctly', 2, None, 'doing-it-correctly'),
("Bayes' Theorem and Ridge and Lasso Regression",
2,
None,
'bayes-theorem-and-ridge-and-lasso-regression'),
('Test Function for what happens with OLS, Ridge and Lasso',
2,
None,
'test-function-for-what-happens-with-ols-ridge-and-lasso'),
("Invoking Bayes' theorem", 2, None, 'invoking-bayes-theorem'),
('Ridge and Bayes', 2, None, 'ridge-and-bayes'),
('Lasso and Bayes', 2, None, 'lasso-and-bayes'),
('Exercise 1: mean values and variances in linear regression',
2,
None,
'exercise-1-mean-values-and-variances-in-linear-regression'),
('Exercise 2: Adding Ridge and Lasso Regression',
2,
None,
'exercise-2-adding-ridge-and-lasso-regression')]}
end of tocinfo -->
<body>
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: {
equationNumbers: { autoNumber: "none" },
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
}
});
</script>
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
<!-- Bootstrap navigation bar -->
<div class="navbar navbar-default navbar-fixed-top">
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="week36-bs.html">Week 36: Statistical interpretation of Linear Regression and Resampling techniques</a>
</div>
<div class="navbar-collapse collapse navbar-responsive-collapse">
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
<ul class="dropdown-menu">
<!-- navigation toc: --> <li><a href="._week36-bs001.html#plans-for-week-36" style="font-size: 80%;">Plans for week 36</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs002.html#summary-from-last-week-and-discussion-of-svd-ridge-and-lasso-regression-with-examples" style="font-size: 80%;">Summary from last Week and discussion of SVD, Ridge and Lasso regression with examples</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs003.html#linear-regression-and-the-svd" style="font-size: 80%;">Linear Regression and the SVD</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs004.html#what-does-it-mean" style="font-size: 80%;">What does it mean?</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs005.html#and-finally-boldsymbol-x-boldsymbol-x-t" style="font-size: 80%;">And finally \( \boldsymbol{X}\boldsymbol{X}^T \)</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs006.html#code-for-svd-and-inversion-of-matrices" style="font-size: 80%;">Code for SVD and Inversion of Matrices</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs007.html#inverse-of-rectangular-matrix" style="font-size: 80%;">Inverse of Rectangular Matrix</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs008.html#ridge-and-lasso-regression" style="font-size: 80%;">Ridge and LASSO Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs009.html#from-ols-to-ridge-and-lasso" style="font-size: 80%;">From OLS to Ridge and Lasso</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs010.html#deriving-the-ridge-regression-equations" style="font-size: 80%;">Deriving the Ridge Regression Equations</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs011.html#note-on-scikit-learn" style="font-size: 80%;">Note on Scikit-Learn</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs012.html#comparison-with-ols" style="font-size: 80%;">Comparison with OLS</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs013.html#svd-analysis" style="font-size: 80%;">SVD analysis</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs014.html#interpreting-the-ridge-results" style="font-size: 80%;">Interpreting the Ridge results</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs015.html#more-interpretations" style="font-size: 80%;">More interpretations</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs016.html#deriving-the-lasso-regression-equations" style="font-size: 80%;">Deriving the Lasso Regression Equations</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs017.html#simple-example-to-illustrate-ordinary-least-squares-ridge-and-lasso-regression" style="font-size: 80%;">Simple example to illustrate Ordinary Least Squares, Ridge and Lasso Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs018.html#ridge-regression" style="font-size: 80%;">Ridge Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs019.html#lasso-regression" style="font-size: 80%;">Lasso Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs020.html#yet-another-example" style="font-size: 80%;">Yet another Example</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs021.html#the-ols-case" style="font-size: 80%;">The OLS case</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs022.html#the-ridge-case" style="font-size: 80%;">The Ridge case</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs023.html#writing-the-cost-function" style="font-size: 80%;">Writing the Cost Function</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs024.html#lasso-case" style="font-size: 80%;">Lasso case</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs025.html#the-first-case" style="font-size: 80%;">The first Case</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs026.html#simple-code-for-solving-the-above-problem" style="font-size: 80%;">Simple code for solving the above problem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs027.html#with-lasso-regression" style="font-size: 80%;">With Lasso Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs028.html#another-example-now-with-a-polynomial-fit" style="font-size: 80%;">Another Example, now with a polynomial fit</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs029.html#to-think-about-first-part" style="font-size: 80%;">To think about, first part</a></li>
<!-- navigation toc: --> <li><a href="#more-thinking" style="font-size: 80%;">More thinking</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs031.html#still-thinking" style="font-size: 80%;">Still thinking</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs032.html#what-does-centering-subtracting-the-mean-values-mean-mathematically" style="font-size: 80%;">What does centering (subtracting the mean values) mean mathematically?</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs033.html#further-manipulations" style="font-size: 80%;">Further Manipulations</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs034.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs035.html#linear-regression-code-intercept-handling-first" style="font-size: 80%;">Linear Regression code, Intercept handling first</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs036.html#code-examples" style="font-size: 80%;">Code Examples</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs037.html#taking-out-the-mean" style="font-size: 80%;">Taking out the mean</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs038.html#friday-september-9" style="font-size: 80%;">Friday September 9</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs039.html#linking-the-regression-analysis-with-a-statistical-interpretation" style="font-size: 80%;">Linking the regression analysis with a statistical interpretation</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs040.html#assumptions-made" style="font-size: 80%;">Assumptions made</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs041.html#expectation-value-and-variance" style="font-size: 80%;">Expectation value and variance</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs042.html#expectation-value-and-variance-for-boldsymbol-beta" style="font-size: 80%;">Expectation value and variance for \( \boldsymbol{\beta} \)</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs043.html#deriving-ols-from-a-probability-distribution" style="font-size: 80%;">Deriving OLS from a probability distribution</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs044.html#independent-and-identically-distrubuted-iid" style="font-size: 80%;">Independent and Identically Distrubuted (iid)</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs045.html#maximum-likelihood-estimation-mle" style="font-size: 80%;">Maximum Likelihood Estimation (MLE)</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs046.html#a-new-cost-function" style="font-size: 80%;">A new Cost Function</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs047.html#more-basic-statistics-and-bayes-theorem" style="font-size: 80%;">More basic Statistics and Bayes' theorem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs048.html#marginal-probability" style="font-size: 80%;">Marginal Probability</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs049.html#conditional-probability" style="font-size: 80%;">Conditional Probability</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs050.html#bayes-theorem" style="font-size: 80%;">Bayes' Theorem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs051.html#interpretations-of-bayes-theorem" style="font-size: 80%;">Interpretations of Bayes' Theorem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs052.html#example-of-usage-of-bayes-theorem" style="font-size: 80%;">Example of Usage of Bayes' theorem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs053.html#doing-it-correctly" style="font-size: 80%;">Doing it correctly</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs054.html#bayes-theorem-and-ridge-and-lasso-regression" style="font-size: 80%;">Bayes' Theorem and Ridge and Lasso Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs055.html#test-function-for-what-happens-with-ols-ridge-and-lasso" style="font-size: 80%;">Test Function for what happens with OLS, Ridge and Lasso</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs056.html#invoking-bayes-theorem" style="font-size: 80%;">Invoking Bayes' theorem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs057.html#ridge-and-bayes" style="font-size: 80%;">Ridge and Bayes</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs058.html#lasso-and-bayes" style="font-size: 80%;">Lasso and Bayes</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs058.html#exercise-1-mean-values-and-variances-in-linear-regression" style="font-size: 80%;">Exercise 1: mean values and variances in linear regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs058.html#exercise-2-adding-ridge-and-lasso-regression" style="font-size: 80%;">Exercise 2: Adding Ridge and Lasso Regression</a></li>
</ul>
</li>
</ul>
</div>
</div>
</div> <!-- end of navigation bar -->
<div class="container">
<p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
<a name="part0030"></a>
<!-- !split -->
<h2 id="more-thinking" class="anchor">More thinking </h2>
<p>If our predictors&#160;represent different&#160;scales, then it is important to
standardize the design matrix \( \boldsymbol{X} \) by subtracting the mean of each
column from the corresponding column and dividing the column with its
standard deviation. Most machine learning libraries do this as a default. This means that if you compare your code with the results from a given library,
the results may differ.
</p>
<p>The
<a href="https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html" target="_self">Standadscaler</a>
function in <b>Scikit-Learn</b> does this for us. For the data sets we
have been studying in our various examples, the data are in many cases
already scaled and there is no need to scale them. You as a user of different machine learning algorithms, should always perform a
survey of your data, with a critical assessment of them in case you need to scale the data.
</p>
<p>If you need to scale the data, not doing so will give an <em>unfair</em>
penalization of the parameters since their magnitude&#160;depends on the
scale of their corresponding&#160;predictor.
</p>
<p>Suppose as an example that you
you have an input&#160;variable given by the heights of different persons.
Human height might be measured in inches or meters or
kilometers. If measured in kilometers, a&#160;standard linear regression
model with this predictor would probably give a much bigger
coefficient term, than if measured in millimeters.
This can clearly lead to problems in evaluating the cost/loss functions.
</p>
<p>
<!-- navigation buttons at the bottom of the page -->
<ul class="pagination">
<li><a href="._week36-bs029.html">&laquo;</a></li>
<li><a href="._week36-bs000.html">1</a></li>
<li><a href="">...</a></li>
<li><a href="._week36-bs022.html">23</a></li>
<li><a href="._week36-bs023.html">24</a></li>
<li><a href="._week36-bs024.html">25</a></li>
<li><a href="._week36-bs025.html">26</a></li>
<li><a href="._week36-bs026.html">27</a></li>
<li><a href="._week36-bs027.html">28</a></li>
<li><a href="._week36-bs028.html">29</a></li>
<li><a href="._week36-bs029.html">30</a></li>
<li class="active"><a href="._week36-bs030.html">31</a></li>
<li><a href="._week36-bs031.html">32</a></li>
<li><a href="._week36-bs032.html">33</a></li>
<li><a href="._week36-bs033.html">34</a></li>
<li><a href="._week36-bs034.html">35</a></li>
<li><a href="._week36-bs035.html">36</a></li>
<li><a href="._week36-bs036.html">37</a></li>
<li><a href="._week36-bs037.html">38</a></li>
<li><a href="._week36-bs038.html">39</a></li>
<li><a href="._week36-bs039.html">40</a></li>
<li><a href="">...</a></li>
<li><a href="._week36-bs058.html">59</a></li>
<li><a href="._week36-bs031.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
</div> <!-- end container -->
<!-- include javascript, jQuery *first* -->
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
<!-- Bootstrap footer
<footer>
<a href="https://..."><img width="250" align=right src="https://..."></a>
</footer>
-->
<center style="font-size:80%">
<!-- copyright only on the titlepage -->
</center>
</body>
</html>