Morten Hjorth-Jensen
2812 lines
156 KiB
HTML
2812 lines
156 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week37.do.txt --pygments_html_style=perldoc --html_style=solarized3 --html_links_in_new_window --html_output=week37-solarized --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 37: Statistical interpretations and Resampling Methods">
|
|
<title>Week 37: Statistical interpretations and Resampling Methods</title>
|
|
<link href="https://cdn.rawgit.com/doconce/doconce/master/bundled/html_styles/style_solarized_box/css/solarized_light_code.css" rel="stylesheet" type="text/css" title="light"/>
|
|
<script src="https://cdn.rawgit.com/doconce/doconce/master/bundled/html_styles/style_solarized_box/js/highlight.pack.js"></script>
|
|
<script>hljs.initHighlightingOnLoad();</script>
|
|
<link href="https://thomasf.github.io/solarized-css/solarized-light.min.css" rel="stylesheet">
|
|
<style type="text/css">
|
|
h1 {color: #b58900;} /* yellow */
|
|
/* h1 {color: #cb4b16;} orange */
|
|
/* h1 {color: #d33682;} magenta, the original choice of thomasf */
|
|
code { padding: 0px; background-color: inherit; }
|
|
pre {
|
|
border: 0pt solid #93a1a1;
|
|
box-shadow: none;
|
|
}
|
|
.alert-text-small { font-size: 80%; }
|
|
.alert-text-large { font-size: 130%; }
|
|
.alert-text-normal { font-size: 90%; }
|
|
.alert {
|
|
padding:8px 35px 8px 14px; margin-bottom:18px;
|
|
text-shadow:0 1px 0 rgba(255,255,255,0.5);
|
|
border:1px solid #93a1a1;
|
|
border-radius: 4px;
|
|
-webkit-border-radius: 4px;
|
|
-moz-border-radius: 4px;
|
|
color: #555;
|
|
background-color: #eee8d5;
|
|
background-position: 10px 5px;
|
|
background-repeat: no-repeat;
|
|
background-size: 38px;
|
|
padding-left: 55px;
|
|
width: 75%;
|
|
}
|
|
.alert-block {padding-top:14px; padding-bottom:14px}
|
|
.alert-block > p, .alert-block > ul {margin-bottom:1em}
|
|
.alert li {margin-top: 1em}
|
|
.alert-block p+p {margin-top:5px}
|
|
.alert-notice { background-image: url(https://cdn.rawgit.com/doconce/doconce/master/bundled/html_images/small_yellow_notice.png); }
|
|
.alert-summary { background-image:url(https://cdn.rawgit.com/doconce/doconce/master/bundled/html_images/small_yellow_summary.png); }
|
|
.alert-warning { background-image: url(https://cdn.rawgit.com/doconce/doconce/master/bundled/html_images/small_yellow_warning.png); }
|
|
.alert-question {background-image:url(https://cdn.rawgit.com/doconce/doconce/master/bundled/html_images/small_yellow_question.png); }
|
|
div { text-align: justify; text-justify: inter-word; }
|
|
.tab {
|
|
padding-left: 1.5em;
|
|
}
|
|
div.toc p,a {
|
|
line-height: 1.3;
|
|
margin-top: 1.1;
|
|
margin-bottom: 1.1;
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plans for week 37, lecture Monday',
|
|
2,
|
|
None,
|
|
'plans-for-week-37-lecture-monday'),
|
|
('Readings and Videos:', 2, None, 'readings-and-videos'),
|
|
('Material for lecture Monday September 8',
|
|
2,
|
|
None,
|
|
'material-for-lecture-monday-september-8'),
|
|
('Gradient descent and revisiting Ordinary Least Squares from '
|
|
'last week',
|
|
2,
|
|
None,
|
|
'gradient-descent-and-revisiting-ordinary-least-squares-from-last-week'),
|
|
('Gradient descent example', 2, None, 'gradient-descent-example'),
|
|
('The derivative of the cost/loss function',
|
|
2,
|
|
None,
|
|
'the-derivative-of-the-cost-loss-function'),
|
|
('The Hessian matrix', 2, None, 'the-hessian-matrix'),
|
|
('Simple program', 2, None, 'simple-program'),
|
|
('Gradient Descent Example', 2, None, 'gradient-descent-example'),
|
|
('Gradient descent and Ridge',
|
|
2,
|
|
None,
|
|
'gradient-descent-and-ridge'),
|
|
('The Hessian matrix for Ridge Regression',
|
|
2,
|
|
None,
|
|
'the-hessian-matrix-for-ridge-regression'),
|
|
('Program example for gradient descent with Ridge Regression',
|
|
2,
|
|
None,
|
|
'program-example-for-gradient-descent-with-ridge-regression'),
|
|
('Using gradient descent methods, limitations',
|
|
2,
|
|
None,
|
|
'using-gradient-descent-methods-limitations'),
|
|
('Momentum based GD', 2, None, 'momentum-based-gd'),
|
|
('Improving gradient descent with momentum',
|
|
2,
|
|
None,
|
|
'improving-gradient-descent-with-momentum'),
|
|
('Same code but now with momentum gradient descent',
|
|
2,
|
|
None,
|
|
'same-code-but-now-with-momentum-gradient-descent'),
|
|
('Overview video on Stochastic Gradient Descent (SGD)',
|
|
2,
|
|
None,
|
|
'overview-video-on-stochastic-gradient-descent-sgd'),
|
|
('Batches and mini-batches', 2, None, 'batches-and-mini-batches'),
|
|
('Pros and cons', 2, None, 'pros-and-cons'),
|
|
('Convergence rates', 2, None, 'convergence-rates'),
|
|
('Accuracy', 2, None, 'accuracy'),
|
|
('Stochastic Gradient Descent (SGD)',
|
|
2,
|
|
None,
|
|
'stochastic-gradient-descent-sgd'),
|
|
('Stochastic Gradient Descent',
|
|
2,
|
|
None,
|
|
'stochastic-gradient-descent'),
|
|
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
|
('SGD example', 2, None, 'sgd-example'),
|
|
('The gradient step', 2, None, 'the-gradient-step'),
|
|
('Simple example code', 2, None, 'simple-example-code'),
|
|
('When do we stop?', 2, None, 'when-do-we-stop'),
|
|
('Slightly different approach',
|
|
2,
|
|
None,
|
|
'slightly-different-approach'),
|
|
('Time decay rate', 2, None, 'time-decay-rate'),
|
|
('Code with a Number of Minibatches which varies',
|
|
2,
|
|
None,
|
|
'code-with-a-number-of-minibatches-which-varies'),
|
|
('Replace or not', 2, None, 'replace-or-not'),
|
|
('Second moment of the gradient',
|
|
2,
|
|
None,
|
|
'second-moment-of-the-gradient'),
|
|
('Challenge: Choosing a Fixed Learning Rate',
|
|
2,
|
|
None,
|
|
'challenge-choosing-a-fixed-learning-rate'),
|
|
('Motivation for Adaptive Step Sizes',
|
|
2,
|
|
None,
|
|
'motivation-for-adaptive-step-sizes'),
|
|
('AdaGrad algorithm, taken from "Goodfellow et '
|
|
'al":"https://www.deeplearningbook.org/contents/optimization.html"',
|
|
2,
|
|
None,
|
|
'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'),
|
|
('Derivation of the AdaGrad Algorithm',
|
|
2,
|
|
None,
|
|
'derivation-of-the-adagrad-algorithm'),
|
|
('AdaGrad Update Rule Derivation',
|
|
2,
|
|
None,
|
|
'adagrad-update-rule-derivation'),
|
|
('AdaGrad Properties', 2, None, 'adagrad-properties'),
|
|
('RMSProp: Adaptive Learning Rates',
|
|
2,
|
|
None,
|
|
'rmsprop-adaptive-learning-rates'),
|
|
('RMSProp algorithm, taken from "Goodfellow et '
|
|
'al":"https://www.deeplearningbook.org/contents/optimization.html"',
|
|
2,
|
|
None,
|
|
'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'),
|
|
('Adam Optimizer', 2, None, 'adam-optimizer'),
|
|
('"ADAM optimizer":"https://arxiv.org/abs/1412.6980"',
|
|
2,
|
|
None,
|
|
'adam-optimizer-https-arxiv-org-abs-1412-6980'),
|
|
('Why Combine Momentum and RMSProp?',
|
|
2,
|
|
None,
|
|
'why-combine-momentum-and-rmsprop'),
|
|
('Adam: Exponential Moving Averages (Moments)',
|
|
2,
|
|
None,
|
|
'adam-exponential-moving-averages-moments'),
|
|
('Adam: Bias Correction', 2, None, 'adam-bias-correction'),
|
|
('Adam: Update Rule Derivation',
|
|
2,
|
|
None,
|
|
'adam-update-rule-derivation'),
|
|
('Adam vs. AdaGrad and RMSProp',
|
|
2,
|
|
None,
|
|
'adam-vs-adagrad-and-rmsprop'),
|
|
('Adaptivity Across Dimensions',
|
|
2,
|
|
None,
|
|
'adaptivity-across-dimensions'),
|
|
('ADAM algorithm, taken from "Goodfellow et '
|
|
'al":"https://www.deeplearningbook.org/contents/optimization.html"',
|
|
2,
|
|
None,
|
|
'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'),
|
|
('Algorithms and codes for Adagrad, RMSprop and Adam',
|
|
2,
|
|
None,
|
|
'algorithms-and-codes-for-adagrad-rmsprop-and-adam'),
|
|
('Practical tips', 2, None, 'practical-tips'),
|
|
('Sneaking in automatic differentiation using Autograd',
|
|
2,
|
|
None,
|
|
'sneaking-in-automatic-differentiation-using-autograd'),
|
|
('Same code but now with momentum gradient descent',
|
|
2,
|
|
None,
|
|
'same-code-but-now-with-momentum-gradient-descent'),
|
|
('Including Stochastic Gradient Descent with Autograd',
|
|
2,
|
|
None,
|
|
'including-stochastic-gradient-descent-with-autograd'),
|
|
('Same code but now with momentum gradient descent',
|
|
2,
|
|
None,
|
|
'same-code-but-now-with-momentum-gradient-descent'),
|
|
("But none of these can compete with Newton's method",
|
|
2,
|
|
None,
|
|
'but-none-of-these-can-compete-with-newton-s-method'),
|
|
('Similar (second order function now) problem but now with '
|
|
'AdaGrad',
|
|
2,
|
|
None,
|
|
'similar-second-order-function-now-problem-but-now-with-adagrad'),
|
|
('RMSprop for adaptive learning rate with Stochastic Gradient '
|
|
'Descent',
|
|
2,
|
|
None,
|
|
'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'),
|
|
('And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"',
|
|
2,
|
|
None,
|
|
'and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf'),
|
|
('Material for the lab sessions',
|
|
2,
|
|
None,
|
|
'material-for-the-lab-sessions'),
|
|
('Reminder on different scaling methods',
|
|
2,
|
|
None,
|
|
'reminder-on-different-scaling-methods'),
|
|
('Functionality in Scikit-Learn',
|
|
2,
|
|
None,
|
|
'functionality-in-scikit-learn'),
|
|
('More preprocessing', 2, None, 'more-preprocessing'),
|
|
('Frequently used scaling functions',
|
|
2,
|
|
None,
|
|
'frequently-used-scaling-functions')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "AMS" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- ------------------- main content ---------------------- -->
|
|
<center>
|
|
<h1>Week 37: Statistical interpretations and Resampling Methods</h1>
|
|
</center> <!-- document title -->
|
|
|
|
<!-- author(s): Morten Hjorth-Jensen -->
|
|
<center>
|
|
<b>Morten Hjorth-Jensen</b>
|
|
</center>
|
|
<!-- institution -->
|
|
<center>
|
|
<b>Department of Physics, University of Oslo, Norway</b>
|
|
</center>
|
|
<br>
|
|
<center>
|
|
<h4>September 8-12, 2025</h4>
|
|
</center> <!-- date -->
|
|
<br>
|
|
|
|
<!-- todo add link to videos and add link to Van Wieringens notes -->
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="plans-for-week-37-lecture-monday">Plans for week 37, lecture Monday </h2>
|
|
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b>Plans and material for the lecture on Monday September 8</b>
|
|
<p>
|
|
<p>The family of gradient descent methods</p>
|
|
<ol>
|
|
<li> Plain gradient descent (constant learning rate), reminder from last week with examples using OLS and Ridge</li>
|
|
<li> Improving gradient descent with momentum</li>
|
|
<li> Introducing stochastic gradient descent</li>
|
|
<li> More advanced updates of the learning rate: ADAgrad, RMSprop and ADAM
|
|
<!-- * <a href="https://youtu.be/omLmp_kkie0" target="_blank">Video of Lecture</a> -->
|
|
<!-- * <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2024/NotesSeptember9.pdf" target="_blank">Whiteboard notes</a> --></li>
|
|
</ol>
|
|
</div>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="readings-and-videos">Readings and Videos: </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
<ol>
|
|
<li> Recommended: Goodfellow et al, Deep Learning, introduction to gradient descent, see sections 4.3-4.5 at <a href="https://www.deeplearningbook.org/contents/numerical.html" target="_blank"><tt>https://www.deeplearningbook.org/contents/numerical.html</tt></a> and chapter 8.3-8.5 at <a href="https://www.deeplearningbook.org/contents/optimization.html" target="_blank"><tt>https://www.deeplearningbook.org/contents/optimization.html</tt></a></li>
|
|
<li> Rashcka et al, pages 37-44 and pages 278-283 with focus on linear regression.</li>
|
|
<li> Video on gradient descent at <a href="https://www.youtube.com/watch?v=sDv4f4s2SB8" target="_blank"><tt>https://www.youtube.com/watch?v=sDv4f4s2SB8</tt></a></li>
|
|
<li> Video on Stochastic gradient descent at <a href="https://www.youtube.com/watch?v=vMh0zPT0tLI" target="_blank"><tt>https://www.youtube.com/watch?v=vMh0zPT0tLI</tt></a></li>
|
|
</ol>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="material-for-lecture-monday-september-8">Material for lecture Monday September 8 </h2>
|
|
|
|
<!-- !split -->
|
|
<h2 id="gradient-descent-and-revisiting-ordinary-least-squares-from-last-week">Gradient descent and revisiting Ordinary Least Squares from last week </h2>
|
|
|
|
<p>Last week we started with linear regression as a case study for the gradient descent
|
|
methods. Linear regression is a great test case for the gradient
|
|
descent methods discussed in the lectures since it has several
|
|
desirable properties such as:
|
|
</p>
|
|
|
|
<ol>
|
|
<li> An analytical solution (recall homework sets for week 35).</li>
|
|
<li> The gradient can be computed analytically.</li>
|
|
<li> The cost function is convex which guarantees that gradient descent converges for small enough learning rates</li>
|
|
</ol>
|
|
<p>We revisit an example similar to what we had in the first homework set. We have a function of the type</p>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
x = <span style="color: #B452CD">2</span>*np.random.rand(m,<span style="color: #B452CD">1</span>)
|
|
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(m,<span style="color: #B452CD">1</span>)
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<p>with \( x_i \in [0,1] \) is chosen randomly using a uniform distribution. Additionally we have a stochastic noise chosen according to a normal distribution \( \cal {N}(0,1) \).
|
|
The linear regression model is given by
|
|
</p>
|
|
$$
|
|
h_\theta(x) = \boldsymbol{y} = \theta_0 + \theta_1 x,
|
|
$$
|
|
|
|
<p>such that </p>
|
|
$$
|
|
\boldsymbol{y}_i = \theta_0 + \theta_1 x_i.
|
|
$$
|
|
|
|
|
|
<!-- !split -->
|
|
<h2 id="gradient-descent-example">Gradient descent example </h2>
|
|
|
|
<p>Let \( \mathbf{y} = (y_1,\cdots,y_n)^T \), \( \mathbf{\boldsymbol{y}} = (\boldsymbol{y}_1,\cdots,\boldsymbol{y}_n)^T \) and \( \theta = (\theta_0, \theta_1)^T \)</p>
|
|
|
|
<p>It is convenient to write \( \mathbf{\boldsymbol{y}} = X\theta \) where \( X \in \mathbb{R}^{100 \times 2} \) is the design matrix given by (we keep the intercept here)</p>
|
|
$$
|
|
X \equiv \begin{bmatrix}
|
|
1 & x_1 \\
|
|
\vdots & \vdots \\
|
|
1 & x_{100} & \\
|
|
\end{bmatrix}.
|
|
$$
|
|
|
|
<p>The cost/loss/risk function is given by </p>
|
|
$$
|
|
C(\theta) = \frac{1}{n}||X\theta-\mathbf{y}||_{2}^{2} = \frac{1}{n}\sum_{i=1}^{100}\left[ (\theta_0 + \theta_1 x_i)^2 - 2 y_i (\theta_0 + \theta_1 x_i) + y_i^2\right]
|
|
$$
|
|
|
|
<p>and we want to find \( \theta \) such that \( C(\theta) \) is minimized.</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="the-derivative-of-the-cost-loss-function">The derivative of the cost/loss function </h2>
|
|
|
|
<p>Computing \( \partial C(\theta) / \partial \theta_0 \) and \( \partial C(\theta) / \partial \theta_1 \) we can show that the gradient can be written as</p>
|
|
$$
|
|
\nabla_{\theta} C(\theta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\theta_0+\theta_1x_i-y_i\right) \\
|
|
\sum_{i=1}^{100}\left( x_i (\theta_0+\theta_1x_i)-y_ix_i\right) \\
|
|
\end{bmatrix} = \frac{2}{n}X^T(X\theta - \mathbf{y}),
|
|
$$
|
|
|
|
<p>where \( X \) is the design matrix defined above.</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="the-hessian-matrix">The Hessian matrix </h2>
|
|
<p>The Hessian matrix of \( C(\theta) \) is given by </p>
|
|
$$
|
|
\boldsymbol{H} \equiv \begin{bmatrix}
|
|
\frac{\partial^2 C(\theta)}{\partial \theta_0^2} & \frac{\partial^2 C(\theta)}{\partial \theta_0 \partial \theta_1} \\
|
|
\frac{\partial^2 C(\theta)}{\partial \theta_0 \partial \theta_1} & \frac{\partial^2 C(\theta)}{\partial \theta_1^2} & \\
|
|
\end{bmatrix} = \frac{2}{n}X^T X.
|
|
$$
|
|
|
|
<p>This result implies that \( C(\theta) \) is a convex function since the matrix \( X^T X \) always is positive semi-definite.</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="simple-program">Simple program </h2>
|
|
|
|
<p>We can now write a program that minimizes \( C(\theta) \) using the gradient descent method with a constant learning rate \( \eta \) according to </p>
|
|
$$
|
|
\theta_{k+1} = \theta_k - \eta \nabla_\theta C(\theta_k), \ k=0,1,\cdots
|
|
$$
|
|
|
|
<p>We can use the expression we computed for the gradient and let use a
|
|
\( \theta_0 \) be chosen randomly and let \( \eta = 0.001 \). Stop iterating
|
|
when \( ||\nabla_\theta C(\theta_k) || \leq \epsilon = 10^{-8} \). <b>Note that the code below does not include the latter stop criterion</b>.
|
|
</p>
|
|
|
|
<p>And finally we can compare our solution for \( \theta \) with the analytic result given by
|
|
\( \theta= (X^TX)^{-1} X^T \mathbf{y} \).
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="gradient-descent-example">Gradient Descent Example </h2>
|
|
|
|
<p>Here our simple example</p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #228B22"># Importing various packages</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">mpl_toolkits.mplot3d</span> <span style="color: #8B008B; font-weight: bold">import</span> Axes3D
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">matplotlib</span> <span style="color: #8B008B; font-weight: bold">import</span> cm
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">matplotlib.ticker</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearLocator, FormatStrFormatter
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">sys</span>
|
|
|
|
<span style="color: #228B22"># the number of datapoints</span>
|
|
n = <span style="color: #B452CD">100</span>
|
|
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
|
|
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
|
|
|
|
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
|
|
<span style="color: #228B22"># Hessian matrix</span>
|
|
H = (<span style="color: #B452CD">2.0</span>/n)* X.T @ X
|
|
<span style="color: #228B22"># Get the eigenvalues</span>
|
|
EigValues, EigVectors = np.linalg.eig(H)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"Eigenvalues of Hessian Matrix:{</span>EigValues<span style="color: #CD5555">}"</span>)
|
|
|
|
theta_linreg = np.linalg.inv(X.T @ X) @ X.T @ y
|
|
<span style="color: #658b00">print</span>(theta_linreg)
|
|
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
|
|
|
|
eta = <span style="color: #B452CD">1.0</span>/np.max(EigValues)
|
|
Niterations = <span style="color: #B452CD">1000</span>
|
|
|
|
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
|
|
gradient = (<span style="color: #B452CD">2.0</span>/n)*X.T @ (X @ theta-y)
|
|
theta -= eta*gradient
|
|
|
|
<span style="color: #658b00">print</span>(theta)
|
|
xnew = np.array([[<span style="color: #B452CD">0</span>],[<span style="color: #B452CD">2</span>]])
|
|
xbnew = np.c_[np.ones((<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)), xnew]
|
|
ypredict = xbnew.dot(theta)
|
|
ypredict2 = xbnew.dot(theta_linreg)
|
|
plt.plot(xnew, ypredict, <span style="color: #CD5555">"r-"</span>)
|
|
plt.plot(xnew, ypredict2, <span style="color: #CD5555">"b-"</span>)
|
|
plt.plot(x, y ,<span style="color: #CD5555">'ro'</span>)
|
|
plt.axis([<span style="color: #B452CD">0</span>,<span style="color: #B452CD">2.0</span>,<span style="color: #B452CD">0</span>, <span style="color: #B452CD">15.0</span>])
|
|
plt.xlabel(<span style="color: #CD5555">r'$x$'</span>)
|
|
plt.ylabel(<span style="color: #CD5555">r'$y$'</span>)
|
|
plt.title(<span style="color: #CD5555">r'Gradient descent example'</span>)
|
|
plt.show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split -->
|
|
<h2 id="gradient-descent-and-ridge">Gradient descent and Ridge </h2>
|
|
|
|
<p>We have also discussed Ridge regression where the loss function contains a regularized term given by the \( L_2 \) norm of \( \theta \), </p>
|
|
$$
|
|
C_{\text{ridge}}(\theta) = \frac{1}{n}||X\theta -\mathbf{y}||^2 + \lambda ||\theta||^2, \ \lambda \geq 0.
|
|
$$
|
|
|
|
<p>In order to minimize \( C_{\text{ridge}}(\theta) \) using GD we adjust the gradient as follows </p>
|
|
$$
|
|
\nabla_\theta C_{\text{ridge}}(\theta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\theta_0+\theta_1x_i-y_i\right) \\
|
|
\sum_{i=1}^{100}\left( x_i (\theta_0+\theta_1x_i)-y_ix_i\right) \\
|
|
\end{bmatrix} + 2\lambda\begin{bmatrix} \theta_0 \\ \theta_1\end{bmatrix} = 2 (\frac{1}{n}X^T(X\theta - \mathbf{y})+\lambda \theta).
|
|
$$
|
|
|
|
<p>We can easily extend our program to minimize \( C_{\text{ridge}}(\theta) \) using gradient descent and compare with the analytical solution given by </p>
|
|
$$
|
|
\theta_{\text{ridge}} = \left(X^T X + n\lambda I_{2 \times 2} \right)^{-1} X^T \mathbf{y}.
|
|
$$
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="the-hessian-matrix-for-ridge-regression">The Hessian matrix for Ridge Regression </h2>
|
|
<p>The Hessian matrix of Ridge Regression for our simple example is given by </p>
|
|
$$
|
|
\boldsymbol{H} \equiv \begin{bmatrix}
|
|
\frac{\partial^2 C(\theta)}{\partial \theta_0^2} & \frac{\partial^2 C(\theta)}{\partial \theta_0 \partial \theta_1} \\
|
|
\frac{\partial^2 C(\theta)}{\partial \theta_0 \partial \theta_1} & \frac{\partial^2 C(\theta)}{\partial \theta_1^2} & \\
|
|
\end{bmatrix} = \frac{2}{n}X^T X+2\lambda\boldsymbol{I}.
|
|
$$
|
|
|
|
<p>This implies that the Hessian matrix is positive definite, hence the stationary point is a
|
|
minimum.
|
|
Note that the Ridge cost function is convex being a sum of two convex
|
|
functions. Therefore, the stationary point is a global
|
|
minimum of this function.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="program-example-for-gradient-descent-with-ridge-regression">Program example for gradient descent with Ridge Regression </h2>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">mpl_toolkits.mplot3d</span> <span style="color: #8B008B; font-weight: bold">import</span> Axes3D
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">matplotlib</span> <span style="color: #8B008B; font-weight: bold">import</span> cm
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">matplotlib.ticker</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearLocator, FormatStrFormatter
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">sys</span>
|
|
|
|
<span style="color: #228B22"># the number of datapoints</span>
|
|
n = <span style="color: #B452CD">100</span>
|
|
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
|
|
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
|
|
|
|
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
|
|
XT_X = X.T @ X
|
|
|
|
<span style="color: #228B22">#Ridge parameter lambda</span>
|
|
lmbda = <span style="color: #B452CD">0.001</span>
|
|
Id = n*lmbda* np.eye(XT_X.shape[<span style="color: #B452CD">0</span>])
|
|
|
|
<span style="color: #228B22"># Hessian matrix</span>
|
|
H = (<span style="color: #B452CD">2.0</span>/n)* XT_X+<span style="color: #B452CD">2</span>*lmbda* np.eye(XT_X.shape[<span style="color: #B452CD">0</span>])
|
|
<span style="color: #228B22"># Get the eigenvalues</span>
|
|
EigValues, EigVectors = np.linalg.eig(H)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"Eigenvalues of Hessian Matrix:{</span>EigValues<span style="color: #CD5555">}"</span>)
|
|
|
|
|
|
theta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y
|
|
<span style="color: #658b00">print</span>(theta_linreg)
|
|
<span style="color: #228B22"># Start plain gradient descent</span>
|
|
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
|
|
|
|
eta = <span style="color: #B452CD">1.0</span>/np.max(EigValues)
|
|
Niterations = <span style="color: #B452CD">100</span>
|
|
|
|
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
|
|
gradients = <span style="color: #B452CD">2.0</span>/n*X.T @ (X @ (theta)-y)+<span style="color: #B452CD">2</span>*lmbda*theta
|
|
theta -= eta*gradients
|
|
|
|
<span style="color: #658b00">print</span>(theta)
|
|
ypredict = X @ theta
|
|
ypredict2 = X @ theta_linreg
|
|
plt.plot(x, ypredict, <span style="color: #CD5555">"r-"</span>)
|
|
plt.plot(x, ypredict2, <span style="color: #CD5555">"b-"</span>)
|
|
plt.plot(x, y ,<span style="color: #CD5555">'ro'</span>)
|
|
plt.axis([<span style="color: #B452CD">0</span>,<span style="color: #B452CD">2.0</span>,<span style="color: #B452CD">0</span>, <span style="color: #B452CD">15.0</span>])
|
|
plt.xlabel(<span style="color: #CD5555">r'$x$'</span>)
|
|
plt.ylabel(<span style="color: #CD5555">r'$y$'</span>)
|
|
plt.title(<span style="color: #CD5555">r'Gradient descent example for Ridge'</span>)
|
|
plt.show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="using-gradient-descent-methods-limitations">Using gradient descent methods, limitations </h2>
|
|
|
|
<ul>
|
|
<li> <b>Gradient descent (GD) finds local minima of our function</b>. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our cost/loss/risk function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance.</li>
|
|
<li> <b>GD is sensitive to initial conditions</b>. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD.</li>
|
|
<li> <b>Gradients are computationally expensive to calculate for large datasets</b>. In many cases in statistics and ML, the cost/loss/risk function is a sum of terms, with one term for each data point. For example, in linear regression, \( E \propto \sum_{i=1}^n (y_i - \mathbf{w}^T\cdot\mathbf{x}_i)^2 \); for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over <em>all</em> \( n \) data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called "mini batches". This has the added benefit of introducing stochasticity into our algorithm.</li>
|
|
<li> <b>GD is very sensitive to choices of learning rates</b>. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would <em>adaptively</em> choose the learning rates to match the landscape.</li>
|
|
<li> <b>GD treats all directions in parameter space uniformly.</b> Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive.</li>
|
|
<li> GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.</li>
|
|
</ul>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="momentum-based-gd">Momentum based GD </h2>
|
|
|
|
<p>We discuss here some simple examples where we introduce what is called
|
|
'memory'about previous steps, or what is normally called momentum
|
|
gradient descent.
|
|
For the mathematical details, see whiteboad notes from lecture on September 8, 2025.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="improving-gradient-descent-with-momentum">Improving gradient descent with momentum </h2>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">import</span> asarray
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">import</span> arange
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy.random</span> <span style="color: #8B008B; font-weight: bold">import</span> rand
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy.random</span> <span style="color: #8B008B; font-weight: bold">import</span> seed
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">matplotlib</span> <span style="color: #8B008B; font-weight: bold">import</span> pyplot
|
|
|
|
<span style="color: #228B22"># objective function</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">objective</span>(x):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> x**<span style="color: #B452CD">2.0</span>
|
|
|
|
<span style="color: #228B22"># derivative of objective function</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">derivative</span>(x):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> x * <span style="color: #B452CD">2.0</span>
|
|
|
|
<span style="color: #228B22"># gradient descent algorithm</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">gradient_descent</span>(objective, derivative, bounds, n_iter, step_size):
|
|
<span style="color: #228B22"># track all solutions</span>
|
|
solutions, scores = <span style="color: #658b00">list</span>(), <span style="color: #658b00">list</span>()
|
|
<span style="color: #228B22"># generate an initial point</span>
|
|
solution = bounds[:, <span style="color: #B452CD">0</span>] + rand(<span style="color: #658b00">len</span>(bounds)) * (bounds[:, <span style="color: #B452CD">1</span>] - bounds[:, <span style="color: #B452CD">0</span>])
|
|
<span style="color: #228B22"># run the gradient descent</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_iter):
|
|
<span style="color: #228B22"># calculate gradient</span>
|
|
gradient = derivative(solution)
|
|
<span style="color: #228B22"># take a step</span>
|
|
solution = solution - step_size * gradient
|
|
<span style="color: #228B22"># evaluate candidate point</span>
|
|
solution_eval = objective(solution)
|
|
<span style="color: #228B22"># store solution</span>
|
|
solutions.append(solution)
|
|
scores.append(solution_eval)
|
|
<span style="color: #228B22"># report progress</span>
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">'>%d f(%s) = %.5f'</span> % (i, solution, solution_eval))
|
|
<span style="color: #8B008B; font-weight: bold">return</span> [solutions, scores]
|
|
|
|
<span style="color: #228B22"># seed the pseudo random number generator</span>
|
|
seed(<span style="color: #B452CD">4</span>)
|
|
<span style="color: #228B22"># define range for input</span>
|
|
bounds = asarray([[-<span style="color: #B452CD">1.0</span>, <span style="color: #B452CD">1.0</span>]])
|
|
<span style="color: #228B22"># define the total iterations</span>
|
|
n_iter = <span style="color: #B452CD">30</span>
|
|
<span style="color: #228B22"># define the step size</span>
|
|
step_size = <span style="color: #B452CD">0.1</span>
|
|
<span style="color: #228B22"># perform the gradient descent search</span>
|
|
solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size)
|
|
<span style="color: #228B22"># sample input range uniformly at 0.1 increments</span>
|
|
inputs = arange(bounds[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>], bounds[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>]+<span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">0.1</span>)
|
|
<span style="color: #228B22"># compute targets</span>
|
|
results = objective(inputs)
|
|
<span style="color: #228B22"># create a line plot of input vs result</span>
|
|
pyplot.plot(inputs, results)
|
|
<span style="color: #228B22"># plot the solutions found</span>
|
|
pyplot.plot(solutions, scores, <span style="color: #CD5555">'.-'</span>, color=<span style="color: #CD5555">'red'</span>)
|
|
<span style="color: #228B22"># show the plot</span>
|
|
pyplot.show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="same-code-but-now-with-momentum-gradient-descent">Same code but now with momentum gradient descent </h2>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">import</span> asarray
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">import</span> arange
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy.random</span> <span style="color: #8B008B; font-weight: bold">import</span> rand
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy.random</span> <span style="color: #8B008B; font-weight: bold">import</span> seed
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">matplotlib</span> <span style="color: #8B008B; font-weight: bold">import</span> pyplot
|
|
|
|
<span style="color: #228B22"># objective function</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">objective</span>(x):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> x**<span style="color: #B452CD">2.0</span>
|
|
|
|
<span style="color: #228B22"># derivative of objective function</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">derivative</span>(x):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> x * <span style="color: #B452CD">2.0</span>
|
|
|
|
<span style="color: #228B22"># gradient descent algorithm</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">gradient_descent</span>(objective, derivative, bounds, n_iter, step_size, momentum):
|
|
<span style="color: #228B22"># track all solutions</span>
|
|
solutions, scores = <span style="color: #658b00">list</span>(), <span style="color: #658b00">list</span>()
|
|
<span style="color: #228B22"># generate an initial point</span>
|
|
solution = bounds[:, <span style="color: #B452CD">0</span>] + rand(<span style="color: #658b00">len</span>(bounds)) * (bounds[:, <span style="color: #B452CD">1</span>] - bounds[:, <span style="color: #B452CD">0</span>])
|
|
<span style="color: #228B22"># keep track of the change</span>
|
|
change = <span style="color: #B452CD">0.0</span>
|
|
<span style="color: #228B22"># run the gradient descent</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_iter):
|
|
<span style="color: #228B22"># calculate gradient</span>
|
|
gradient = derivative(solution)
|
|
<span style="color: #228B22"># calculate update</span>
|
|
new_change = step_size * gradient + momentum * change
|
|
<span style="color: #228B22"># take a step</span>
|
|
solution = solution - new_change
|
|
<span style="color: #228B22"># save the change</span>
|
|
change = new_change
|
|
<span style="color: #228B22"># evaluate candidate point</span>
|
|
solution_eval = objective(solution)
|
|
<span style="color: #228B22"># store solution</span>
|
|
solutions.append(solution)
|
|
scores.append(solution_eval)
|
|
<span style="color: #228B22"># report progress</span>
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">'>%d f(%s) = %.5f'</span> % (i, solution, solution_eval))
|
|
<span style="color: #8B008B; font-weight: bold">return</span> [solutions, scores]
|
|
|
|
<span style="color: #228B22"># seed the pseudo random number generator</span>
|
|
seed(<span style="color: #B452CD">4</span>)
|
|
<span style="color: #228B22"># define range for input</span>
|
|
bounds = asarray([[-<span style="color: #B452CD">1.0</span>, <span style="color: #B452CD">1.0</span>]])
|
|
<span style="color: #228B22"># define the total iterations</span>
|
|
n_iter = <span style="color: #B452CD">30</span>
|
|
<span style="color: #228B22"># define the step size</span>
|
|
step_size = <span style="color: #B452CD">0.1</span>
|
|
<span style="color: #228B22"># define momentum</span>
|
|
momentum = <span style="color: #B452CD">0.3</span>
|
|
<span style="color: #228B22"># perform the gradient descent search with momentum</span>
|
|
solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum)
|
|
<span style="color: #228B22"># sample input range uniformly at 0.1 increments</span>
|
|
inputs = arange(bounds[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>], bounds[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>]+<span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">0.1</span>)
|
|
<span style="color: #228B22"># compute targets</span>
|
|
results = objective(inputs)
|
|
<span style="color: #228B22"># create a line plot of input vs result</span>
|
|
pyplot.plot(inputs, results)
|
|
<span style="color: #228B22"># plot the solutions found</span>
|
|
pyplot.plot(solutions, scores, <span style="color: #CD5555">'.-'</span>, color=<span style="color: #CD5555">'red'</span>)
|
|
<span style="color: #228B22"># show the plot</span>
|
|
pyplot.show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="overview-video-on-stochastic-gradient-descent-sgd">Overview video on Stochastic Gradient Descent (SGD) </h2>
|
|
|
|
<a href="https://www.youtube.com/watch?v=vMh0zPT0tLI&ab_channel=StatQuestwithJoshStarmer" target="_blank">What is Stochastic Gradient Descent</a>
|
|
<p>There are several reasons for using stochastic gradient descent. Some of these are:</p>
|
|
|
|
<ol>
|
|
<li> Efficiency: Updates weights more frequently using a single or a small batch of samples, which speeds up convergence.</li>
|
|
<li> Hopefully avoid Local Minima</li>
|
|
<li> Memory Usage: Requires less memory compared to computing gradients for the entire dataset.</li>
|
|
</ol>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="batches-and-mini-batches">Batches and mini-batches </h2>
|
|
|
|
<p>In gradient descent we compute the cost function and its gradient for all data points we have.</p>
|
|
|
|
<p>In large-scale applications such as the <a href="https://www.image-net.org/challenges/LSVRC/" target="_blank">ILSVRC challenge</a>, the
|
|
training data can have on order of millions of examples. Hence, it
|
|
seems wasteful to compute the full cost function over the entire
|
|
training set in order to perform only a single parameter update. A
|
|
very common approach to addressing this challenge is to compute the
|
|
gradient over batches of the training data. For example, a typical batch could contain some thousand examples from
|
|
an entire training set of several millions. This batch is then used to
|
|
perform a parameter update.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="pros-and-cons">Pros and cons </h2>
|
|
|
|
<ol>
|
|
<li> Speed: SGD is faster than gradient descent because it uses only one training example per iteration, whereas gradient descent requires the entire dataset. This speed advantage becomes more significant as the size of the dataset increases.</li>
|
|
<li> Convergence: Gradient descent has a more predictable convergence behaviour because it uses the average gradient of the entire dataset. In contrast, SGD’s convergence behaviour can be more erratic due to its random sampling of individual training examples.</li>
|
|
<li> Memory: Gradient descent requires more memory than SGD because it must store the entire dataset for each iteration. SGD only needs to store the current training example, making it more memory-efficient.</li>
|
|
</ol>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="convergence-rates">Convergence rates </h2>
|
|
|
|
<ol>
|
|
<li> Stochastic Gradient Descent has a faster convergence rate due to the use of single training examples in each iteration.</li>
|
|
<li> Gradient Descent as a slower convergence rate, as it uses the entire dataset for each iteration.</li>
|
|
</ol>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="accuracy">Accuracy </h2>
|
|
|
|
<p>In general, stochastic Gradient Descent is Less accurate than gradient
|
|
descent, as it calculates the gradient on single examples, which may
|
|
not accurately represent the overall dataset. Gradient Descent is
|
|
more accurate because it uses the average gradient calculated over the
|
|
entire dataset.
|
|
</p>
|
|
|
|
<p>There are other disadvantages to using SGD. The main drawback is that
|
|
its convergence behaviour can be more erratic due to the random
|
|
sampling of individual training examples. This can lead to less
|
|
accurate results, as the algorithm may not converge to the true
|
|
minimum of the cost function. Additionally, the learning rate, which
|
|
determines the step size of each update to the model’s parameters,
|
|
must be carefully chosen to ensure convergence.
|
|
</p>
|
|
|
|
<p>It is however the method of choice in deep learning algorithms where
|
|
SGD is often used in combination with other optimization techniques,
|
|
such as momentum or adaptive learning rates
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="stochastic-gradient-descent-sgd">Stochastic Gradient Descent (SGD) </h2>
|
|
|
|
<p>In stochastic gradient descent, the extreme case is the case where we
|
|
have only one batch, that is we include the whole data set.
|
|
</p>
|
|
|
|
<p>This process is called Stochastic Gradient
|
|
Descent (SGD) (or also sometimes on-line gradient descent). This is
|
|
relatively less common to see because in practice due to vectorized
|
|
code optimizations it can be computationally much more efficient to
|
|
evaluate the gradient for 100 examples, than the gradient for one
|
|
example 100 times. Even though SGD technically refers to using a
|
|
single example at a time to evaluate the gradient, you will hear
|
|
people use the term SGD even when referring to mini-batch gradient
|
|
descent (i.e. mentions of MGD for “Minibatch Gradient Descent”, or BGD
|
|
for “Batch gradient descent” are rare to see), where it is usually
|
|
assumed that mini-batches are used. The size of the mini-batch is a
|
|
hyperparameter but it is not very common to cross-validate or bootstrap it. It is
|
|
usually based on memory constraints (if any), or set to some value,
|
|
e.g. 32, 64 or 128. We use powers of 2 in practice because many
|
|
vectorized operation implementations work faster when their inputs are
|
|
sized in powers of 2.
|
|
</p>
|
|
|
|
<p>In our notes with SGD we mean stochastic gradient descent with mini-batches.</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="stochastic-gradient-descent">Stochastic Gradient Descent </h2>
|
|
|
|
<p>Stochastic gradient descent (SGD) and variants thereof address some of
|
|
the shortcomings of the Gradient descent method discussed above.
|
|
</p>
|
|
|
|
<p>The underlying idea of SGD comes from the observation that the cost
|
|
function, which we want to minimize, can almost always be written as a
|
|
sum over \( n \) data points \( \{\mathbf{x}_i\}_{i=1}^n \),
|
|
</p>
|
|
$$
|
|
C(\mathbf{\theta}) = \sum_{i=1}^n c_i(\mathbf{x}_i,
|
|
\mathbf{\theta}).
|
|
$$
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="computation-of-gradients">Computation of gradients </h2>
|
|
|
|
<p>This in turn means that the gradient can be
|
|
computed as a sum over \( i \)-gradients
|
|
</p>
|
|
$$
|
|
\nabla_\theta C(\mathbf{\theta}) = \sum_i^n \nabla_\theta c_i(\mathbf{x}_i,
|
|
\mathbf{\theta}).
|
|
$$
|
|
|
|
<p>Stochasticity/randomness is introduced by only taking the
|
|
gradient on a subset of the data called minibatches. If there are \( n \)
|
|
data points and the size of each minibatch is \( M \), there will be \( n/M \)
|
|
minibatches. We denote these minibatches by \( B_k \) where
|
|
\( k=1,\cdots,n/M \).
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="sgd-example">SGD example </h2>
|
|
<p>As an example, suppose we have \( 10 \) data points \( (\mathbf{x}_1,\cdots, \mathbf{x}_{10}) \)
|
|
and we choose to have \( M=5 \) minibathces,
|
|
then each minibatch contains two data points. In particular we have
|
|
\( B_1 = (\mathbf{x}_1,\mathbf{x}_2), \cdots, B_5 =
|
|
(\mathbf{x}_9,\mathbf{x}_{10}) \). Note that if you choose \( M=1 \) you
|
|
have only a single batch with all data points and on the other extreme,
|
|
you may choose \( M=n \) resulting in a minibatch for each datapoint, i.e
|
|
\( B_k = \mathbf{x}_k \).
|
|
</p>
|
|
|
|
<p>The idea is now to approximate the gradient by replacing the sum over
|
|
all data points with a sum over the data points in one the minibatches
|
|
picked at random in each gradient descent step
|
|
</p>
|
|
$$
|
|
\nabla_{\theta}
|
|
C(\mathbf{\theta}) = \sum_{i=1}^n \nabla_\theta c_i(\mathbf{x}_i,
|
|
\mathbf{\theta}) \rightarrow \sum_{i \in B_k}^n \nabla_\theta
|
|
c_i(\mathbf{x}_i, \mathbf{\theta}).
|
|
$$
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="the-gradient-step">The gradient step </h2>
|
|
|
|
<p>Thus a gradient descent step now looks like </p>
|
|
$$
|
|
\theta_{j+1} = \theta_j - \eta_j \sum_{i \in B_k}^n \nabla_\theta c_i(\mathbf{x}_i,
|
|
\mathbf{\theta})
|
|
$$
|
|
|
|
<p>where \( k \) is picked at random with equal
|
|
probability from \( [1,n/M] \). An iteration over the number of
|
|
minibathces (n/M) is commonly referred to as an epoch. Thus it is
|
|
typical to choose a number of epochs and for each epoch iterate over
|
|
the number of minibatches, as exemplified in the code below.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="simple-example-code">Simple example code </h2>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
|
|
n = <span style="color: #B452CD">100</span> <span style="color: #228B22">#100 datapoints </span>
|
|
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
|
|
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
|
|
n_epochs = <span style="color: #B452CD">10</span> <span style="color: #228B22">#number of epochs</span>
|
|
|
|
j = <span style="color: #B452CD">0</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #B452CD">1</span>,n_epochs+<span style="color: #B452CD">1</span>):
|
|
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
|
|
k = np.random.randint(m) <span style="color: #228B22">#Pick the k-th minibatch at random</span>
|
|
<span style="color: #228B22">#Compute the gradient using the data in minibatch Bk</span>
|
|
<span style="color: #228B22">#Compute new suggestion for </span>
|
|
j += <span style="color: #B452CD">1</span>
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<p>Taking the gradient only on a subset of the data has two important
|
|
benefits. First, it introduces randomness which decreases the chance
|
|
that our opmization scheme gets stuck in a local minima. Second, if
|
|
the size of the minibatches are small relative to the number of
|
|
datapoints (\( M < n \)), the computation of the gradient is much
|
|
cheaper since we sum over the datapoints in the \( k-th \) minibatch and not
|
|
all \( n \) datapoints.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="when-do-we-stop">When do we stop? </h2>
|
|
|
|
<p>A natural question is when do we stop the search for a new minimum?
|
|
One possibility is to compute the full gradient after a given number
|
|
of epochs and check if the norm of the gradient is smaller than some
|
|
threshold and stop if true. However, the condition that the gradient
|
|
is zero is valid also for local minima, so this would only tell us
|
|
that we are close to a local/global minimum. However, we could also
|
|
evaluate the cost function at this point, store the result and
|
|
continue the search. If the test kicks in at a later stage we can
|
|
compare the values of the cost function and keep the \( \theta \) that
|
|
gave the lowest value.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="slightly-different-approach">Slightly different approach </h2>
|
|
|
|
<p>Another approach is to let the step length \( \eta_j \) depend on the
|
|
number of epochs in such a way that it becomes very small after a
|
|
reasonable time such that we do not move at all. Such approaches are
|
|
also called scaling. There are many such ways to <a href="https://towardsdatascience.com/gradient-descent-the-learning-rate-and-the-importance-of-feature-scaling-6c0b416596e1" target="_blank">scale the learning
|
|
rate</a>
|
|
and <a href="https://www.jmlr.org/papers/volume23/20-1258/20-1258.pdf" target="_blank">discussions here</a>. See
|
|
also
|
|
<a href="https://towardsdatascience.com/learning-rate-schedules-and-adaptive-learning-rate-methods-for-deep-learning-2c8f433990d1" target="_blank"><tt>https://towardsdatascience.com/learning-rate-schedules-and-adaptive-learning-rate-methods-for-deep-learning-2c8f433990d1</tt></a>
|
|
for a discussion of different scaling functions for the learning rate.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="time-decay-rate">Time decay rate </h2>
|
|
|
|
<p>As an example, let \( e = 0,1,2,3,\cdots \) denote the current epoch and let \( t_0, t_1 > 0 \) be two fixed numbers. Furthermore, let \( t = e \cdot m + i \) where \( m \) is the number of minibatches and \( i=0,\cdots,m-1 \). Then the function $$\eta_j(t; t_0, t_1) = \frac{t_0}{t+t_1} $$ goes to zero as the number of epochs gets large. I.e. we start with a step length \( \eta_j (0; t_0, t_1) = t_0/t_1 \) which decays in <em>time</em> \( t \).</p>
|
|
|
|
<p>In this way we can fix the number of epochs, compute \( \theta \) and
|
|
evaluate the cost function at the end. Repeating the computation will
|
|
give a different result since the scheme is random by design. Then we
|
|
pick the final \( \theta \) that gives the lowest value of the cost
|
|
function.
|
|
</p>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">step_length</span>(t,t0,t1):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> t0/(t+t1)
|
|
|
|
n = <span style="color: #B452CD">100</span> <span style="color: #228B22">#100 datapoints </span>
|
|
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
|
|
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
|
|
n_epochs = <span style="color: #B452CD">500</span> <span style="color: #228B22">#number of epochs</span>
|
|
t0 = <span style="color: #B452CD">1.0</span>
|
|
t1 = <span style="color: #B452CD">10</span>
|
|
|
|
eta_j = t0/t1
|
|
j = <span style="color: #B452CD">0</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #B452CD">1</span>,n_epochs+<span style="color: #B452CD">1</span>):
|
|
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
|
|
k = np.random.randint(m) <span style="color: #228B22">#Pick the k-th minibatch at random</span>
|
|
<span style="color: #228B22">#Compute the gradient using the data in minibatch Bk</span>
|
|
<span style="color: #228B22">#Compute new suggestion for theta</span>
|
|
t = epoch*m+i
|
|
eta_j = step_length(t,t0,t1)
|
|
j += <span style="color: #B452CD">1</span>
|
|
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"eta_j after %d epochs: %g"</span> % (n_epochs,eta_j))
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="code-with-a-number-of-minibatches-which-varies">Code with a Number of Minibatches which varies </h2>
|
|
|
|
<p>In the code here we vary the number of mini-batches.</p>
|
|
|
|
<!-- code=text (!bc pycode) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"># Importing various packages
|
|
from math import exp, sqrt
|
|
from random import random, seed
|
|
import numpy as np
|
|
import matplotlib.pyplot as plt
|
|
|
|
n = 100
|
|
x = 2*np.random.rand(n,1)
|
|
y = 4+3*x+np.random.randn(n,1)
|
|
|
|
X = np.c_[np.ones((n,1)), x]
|
|
XT_X = X.T @ X
|
|
theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
|
|
print("Own inversion")
|
|
print(theta_linreg)
|
|
# Hessian matrix
|
|
H = (2.0/n)* XT_X
|
|
EigValues, EigVectors = np.linalg.eig(H)
|
|
print(f"Eigenvalues of Hessian Matrix:{EigValues}")
|
|
|
|
theta = np.random.randn(2,1)
|
|
eta = 1.0/np.max(EigValues)
|
|
Niterations = 1000
|
|
|
|
|
|
for iter in range(Niterations):
|
|
gradients = 2.0/n*X.T @ ((X @ theta)-y)
|
|
theta -= eta*gradients
|
|
print("theta from own gd")
|
|
print(theta)
|
|
|
|
xnew = np.array([[0],[2]])
|
|
Xnew = np.c_[np.ones((2,1)), xnew]
|
|
ypredict = Xnew.dot(theta)
|
|
ypredict2 = Xnew.dot(theta_linreg)
|
|
|
|
n_epochs = 50
|
|
M = 5 #size of each minibatch
|
|
m = int(n/M) #number of minibatches
|
|
t0, t1 = 5, 50
|
|
|
|
def learning_schedule(t):
|
|
return t0/(t+t1)
|
|
|
|
theta = np.random.randn(2,1)
|
|
|
|
for epoch in range(n_epochs):
|
|
# Can you figure out a better way of setting up the contributions to each batch?
|
|
for i in range(m):
|
|
random_index = M*np.random.randint(m)
|
|
xi = X[random_index:random_index+M]
|
|
yi = y[random_index:random_index+M]
|
|
gradients = (2.0/M)* xi.T @ ((xi @ theta)-yi)
|
|
eta = learning_schedule(epoch*m+i)
|
|
theta = theta - eta*gradients
|
|
print("theta from own sdg")
|
|
print(theta)
|
|
|
|
plt.plot(xnew, ypredict, "r-")
|
|
plt.plot(xnew, ypredict2, "b-")
|
|
plt.plot(x, y ,'ro')
|
|
plt.axis([0,2.0,0, 15.0])
|
|
plt.xlabel(r'$x$')
|
|
plt.ylabel(r'$y$')
|
|
plt.title(r'Random numbers ')
|
|
plt.show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="replace-or-not">Replace or not </h2>
|
|
|
|
<p>In the above code, we have use replacement in setting up the
|
|
mini-batches. The discussion
|
|
<a href="https://sebastianraschka.com/faq/docs/sgd-methods.html" target="_blank">here</a> may be
|
|
useful.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="second-moment-of-the-gradient">Second moment of the gradient </h2>
|
|
|
|
<p>In stochastic gradient descent, with and without momentum, we still
|
|
have to specify a schedule for tuning the learning rates \( \eta_t \)
|
|
as a function of time. As discussed in the context of Newton's
|
|
method, this presents a number of dilemmas. The learning rate is
|
|
limited by the steepest direction which can change depending on the
|
|
current position in the landscape. To circumvent this problem, ideally
|
|
our algorithm would keep track of curvature and take large steps in
|
|
shallow, flat directions and small steps in steep, narrow directions.
|
|
Second-order methods accomplish this by calculating or approximating
|
|
the Hessian and normalizing the learning rate by the
|
|
curvature. However, this is very computationally expensive for
|
|
extremely large models. Ideally, we would like to be able to
|
|
adaptively change the step size to match the landscape without paying
|
|
the steep computational price of calculating or approximating
|
|
Hessians.
|
|
</p>
|
|
|
|
<p>During the last decade a number of methods have been introduced that accomplish
|
|
this by tracking not only the gradient, but also the second moment of
|
|
the gradient. These methods include AdaGrad, AdaDelta, Root Mean Squared Propagation (RMS-Prop), and
|
|
<a href="https://arxiv.org/abs/1412.6980" target="_blank">ADAM</a>.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="challenge-choosing-a-fixed-learning-rate">Challenge: Choosing a Fixed Learning Rate </h2>
|
|
<p>A fixed \( \eta \) is hard to get right:</p>
|
|
<ol>
|
|
<li> If \( \eta \) is too large, the updates can overshoot the minimum, causing oscillations or divergence</li>
|
|
<li> If \( \eta \) is too small, convergence is very slow (many iterations to make progress)</li>
|
|
</ol>
|
|
<p>In practice, one often uses trial-and-error or schedules (decaying \( \eta \) over time) to find a workable balance.
|
|
For a function with steep directions and flat directions, a single global \( \eta \) may be inappropriate:
|
|
</p>
|
|
<ol>
|
|
<li> Steep coordinates require a smaller step size to avoid oscillation.</li>
|
|
<li> Flat/shallow coordinates could use a larger step to speed up progress.</li>
|
|
<li> This issue is pronounced in high-dimensional problems with **sparse or varying-scale features** – we need a method to adjust step sizesper feature.</li>
|
|
</ol>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="motivation-for-adaptive-step-sizes">Motivation for Adaptive Step Sizes </h2>
|
|
|
|
<ol>
|
|
<li> Instead of a fixed global \( \eta \), use an <b>adaptive learning rate</b> for each parameter that depends on the history of gradients.</li>
|
|
<li> Parameters that have large accumulated gradient magnitude should get smaller steps (they've been changing a lot), whereas parameters with small or infrequent gradients can have larger relative steps.</li>
|
|
<li> This is especially useful for sparse features: Rarely active features accumulate little gradient, so their learning rate remains comparatively high, ensuring they are not neglected</li>
|
|
<li> Conversely, frequently active features accumulate large gradient sums, and their learning rate automatically decreases, preventing too-large updates</li>
|
|
<li> Several algorithms implement this idea (AdaGrad, RMSProp, AdaDelta, Adam, etc.). We will derive **AdaGrad**, one of the first adaptive methods.</li>
|
|
</ol>
|
|
<h2 id="adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html">AdaGrad algorithm, taken from <a href="https://www.deeplearningbook.org/contents/optimization.html" target="_blank">Goodfellow et al</a> </h2>
|
|
|
|
<br/><br/>
|
|
<center>
|
|
<p><img src="figures/adagrad.png" width="600" align="bottom"></p>
|
|
</center>
|
|
<br/><br/>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="derivation-of-the-adagrad-algorithm">Derivation of the AdaGrad Algorithm </h2>
|
|
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b>Accumulating Gradient History</b>
|
|
<p>
|
|
<ol>
|
|
<li> AdaGrad maintains a running sum of squared gradients for each parameter (coordinate)</li>
|
|
<li> Let \( g_t = \nabla C_{i_t}(x_t) \) be the gradient at step \( t \) (or a subgradient for nondifferentiable cases).</li>
|
|
<li> Initialize \( r_0 = 0 \) (an all-zero vector in \( \mathbb{R}^d \)).</li>
|
|
<li> At each iteration \( t \), update the accumulation:</li>
|
|
</ol>
|
|
$$
|
|
r_t = r_{t-1} + g_t \circ g_t,
|
|
$$
|
|
|
|
<ol>
|
|
<li> Here \( g_t \circ g_t \) denotes element-wise square of the gradient vector. \( g_t^{(j)} = g_{t-1}^{(j)} + (g_{t,j})^2 \) for each parameter \( j \).</li>
|
|
<li> We can view \( H_t = \mathrm{diag}(r_t) \) as a diagonal matrix of past squared gradients. Initially \( H_0 = 0 \).</li>
|
|
</ol>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="adagrad-update-rule-derivation">AdaGrad Update Rule Derivation </h2>
|
|
|
|
<p>We scale the gradient by the inverse square root of the accumulated matrix \( H_t \). The AdaGrad update at step \( t \) is:</p>
|
|
$$
|
|
\theta_{t+1} =\theta_t - \eta H_t^{-1/2} g_t,
|
|
$$
|
|
|
|
<p>where \( H_t^{-1/2} \) is the diagonal matrix with entries \( (r_{t}^{(1)})^{-1/2}, \dots, (r_{t}^{(d)})^{-1/2} \)
|
|
In coordinates, this means each parameter \( j \) has an individual step size:
|
|
</p>
|
|
$$
|
|
\theta_{t+1,j} =\theta_{t,j} -\frac{\eta}{\sqrt{r_{t,j}}}g_{t,j}.
|
|
$$
|
|
|
|
<p>In practice we add a small constant \( \epsilon \) in the denominator for numerical stability to avoid division by zero:</p>
|
|
$$
|
|
\theta_{t+1,j}= \theta_{t,j}-\frac{\eta}{\sqrt{\epsilon + r_{t,j}}}g_{t,j}.
|
|
$$
|
|
|
|
<p>Equivalently, the effective learning rate for parameter \( j \) at time \( t \) is \( \displaystyle \alpha_{t,j} = \frac{\eta}{\sqrt{\epsilon + r_{t,j}}} \). This decreases over time as \( r_{t,j} \) grows.</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="adagrad-properties">AdaGrad Properties </h2>
|
|
|
|
<ol>
|
|
<li> AdaGrad automatically tunes the step size for each parameter. Parameters with more <em>volatile or large gradients</em> get smaller steps, and those with <em>small or infrequent gradients</em> get relatively larger steps</li>
|
|
<li> No manual schedule needed: The accumulation \( r_t \) keeps increasing (or stays the same if gradient is zero), so step sizes \( \eta/\sqrt{r_t} \) are non-increasing. This has a similar effect to a learning rate schedule, but individualized per coordinate.</li>
|
|
<li> Sparse data benefit: For very sparse features, \( r_{t,j} \) grows slowly, so that feature’s parameter retains a higher learning rate for longer, allowing it to make significant updates when it does get a gradient signal</li>
|
|
<li> Convergence: In convex optimization, AdaGrad can be shown to achieve a sub-linear convergence rate comparable to the best fixed learning rate tuned for the problem</li>
|
|
</ol>
|
|
<p>It effectively reduces the need to tune \( \eta \) by hand.</p>
|
|
<ol>
|
|
<li> Limitations: Because \( r_t \) accumulates without bound, AdaGrad’s learning rates can become extremely small over long training, potentially slowing progress. (Later variants like RMSProp, AdaDelta, Adam address this by modifying the accumulation rule.)</li>
|
|
</ol>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="rmsprop-adaptive-learning-rates">RMSProp: Adaptive Learning Rates </h2>
|
|
|
|
<p>Addresses AdaGrad’s diminishing learning rate issue.
|
|
Uses a decaying average of squared gradients (instead of a cumulative sum):
|
|
</p>
|
|
$$
|
|
v_t = \rho v_{t-1} + (1-\rho)(\nabla C(\theta_t))^2,
|
|
$$
|
|
|
|
<p>with \( \rho \) typically \( 0.9 \) (or \( 0.99 \)).</p>
|
|
<ol>
|
|
<li> Update: \( \theta_{t+1} = \theta_t - \frac{\eta}{\sqrt{v_t + \epsilon}} \nabla C(\theta_t) \).</li>
|
|
<li> Recent gradients have more weight, so \( v_t \) adapts to the current landscape.</li>
|
|
<li> Avoids AdaGrad’s “infinite memory” problem – learning rate does not continuously decay to zero.</li>
|
|
</ol>
|
|
<p>RMSProp was first proposed in lecture notes by Geoff Hinton, 2012 – unpublished.)</p>
|
|
<h2 id="rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html">RMSProp algorithm, taken from <a href="https://www.deeplearningbook.org/contents/optimization.html" target="_blank">Goodfellow et al</a> </h2>
|
|
|
|
<br/><br/>
|
|
<center>
|
|
<p><img src="figures/rmsprop.png" width="600" align="bottom"></p>
|
|
</center>
|
|
<br/><br/>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="adam-optimizer">Adam Optimizer </h2>
|
|
|
|
<p>Why combine Momentum and RMSProp? Motivation for Adam: Adaptive Moment Estimation (Adam) was introduced by Kingma an Ba (2014) to combine the benefits of momentum and RMSProp.</p>
|
|
|
|
<ol>
|
|
<li> Fast convergence by smoothing gradients (accelerates in long-term gradient direction).</li>
|
|
<li> Adaptive rates (RMSProp): Per-dimension learning rate scaling for stability (handles different feature scales, sparse gradients).</li>
|
|
<li> Adam uses both: maintains moving averages of both first moment (gradients) and second moment (squared gradients)</li>
|
|
<li> Additionally, includes a mechanism to correct the bias in these moving averages (crucial in early iterations)</li>
|
|
</ol>
|
|
<p><b>Result</b>: Adam is robust, achieves faster convergence with less tuning, and often outperforms SGD (with momentum) in practice.</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="adam-optimizer-https-arxiv-org-abs-1412-6980"><a href="https://arxiv.org/abs/1412.6980" target="_blank">ADAM optimizer</a> </h2>
|
|
|
|
<p>In <a href="https://arxiv.org/abs/1412.6980" target="_blank">ADAM</a>, we keep a running average of
|
|
both the first and second moment of the gradient and use this
|
|
information to adaptively change the learning rate for different
|
|
parameters. The method is efficient when working with large
|
|
problems involving lots data and/or parameters. It is a combination of the
|
|
gradient descent with momentum algorithm and the RMSprop algorithm
|
|
discussed above.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="why-combine-momentum-and-rmsprop">Why Combine Momentum and RMSProp? </h2>
|
|
|
|
<ol>
|
|
<li> Momentum: Fast convergence by smoothing gradients (accelerates in long-term gradient direction).</li>
|
|
<li> Adaptive rates (RMSProp): Per-dimension learning rate scaling for stability (handles different feature scales, sparse gradients).</li>
|
|
<li> Adam uses both: maintains moving averages of both first moment (gradients) and second moment (squared gradients)</li>
|
|
<li> Additionally, includes a mechanism to correct the bias in these moving averages (crucial in early iterations)</li>
|
|
</ol>
|
|
<p>Result: Adam is robust, achieves faster convergence with less tuning, and often outperforms SGD (with momentum) in practice</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="adam-exponential-moving-averages-moments">Adam: Exponential Moving Averages (Moments) </h2>
|
|
<p>Adam maintains two moving averages at each time step \( t \) for each parameter \( w \):</p>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b>First moment (mean) \( m_t \)</b>
|
|
<p>
|
|
<p>The Momentum term</p>
|
|
$$
|
|
m_t = \beta_1m_{t-1} + (1-\beta_1)\, \nabla C(\theta_t),
|
|
$$
|
|
</div>
|
|
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b>Second moment (uncentered variance) \( v_t \)</b>
|
|
<p>
|
|
<p>The RMS term</p>
|
|
$$
|
|
v_t = \beta_2v_{t-1} + (1-\beta_2)(\nabla C(\theta_t))^2,
|
|
$$
|
|
|
|
<p>with typical \( \beta_1 = 0.9 \), \( \beta_2 = 0.999 \). Initialize \( m_0 = 0 \), \( v_0 = 0 \).</p>
|
|
</div>
|
|
|
|
<p> These are <b>biased</b> estimators of the true first and second moment of the gradients, especially at the start (since \( m_0,v_0 \) are zero)</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="adam-bias-correction">Adam: Bias Correction </h2>
|
|
<p>To counteract initialization bias in \( m_t, v_t \), Adam computes bias-corrected estimates</p>
|
|
$$
|
|
\hat{m}_t = \frac{m_t}{1 - \beta_1^t}, \qquad \hat{v}_t = \frac{v_t}{1 - \beta_2^t}.
|
|
$$
|
|
|
|
<ul>
|
|
<li> When \( t \) is small, \( 1-\beta_i^t \approx 0 \), so \( \hat{m}_t, \hat{v}_t \) significantly larger than raw \( m_t, v_t \), compensating for the initial zero bias.</li>
|
|
<li> As \( t \) increases, \( 1-\beta_i^t \to 1 \), and \( \hat{m}_t, \hat{v}_t \) converge to \( m_t, v_t \).</li>
|
|
<li> Bias correction is important for Adam’s stability in early iterations</li>
|
|
</ul>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="adam-update-rule-derivation">Adam: Update Rule Derivation </h2>
|
|
<p>Finally, Adam updates parameters using the bias-corrected moments:</p>
|
|
$$
|
|
\theta_{t+1} =\theta_t -\frac{\alpha}{\sqrt{\hat{v}_t} + \epsilon}\hat{m}_t,
|
|
$$
|
|
|
|
<p>where \( \epsilon \) is a small constant (e.g. \( 10^{-8} \)) to prevent division by zero.
|
|
Breaking it down:
|
|
</p>
|
|
<ol>
|
|
<li> Compute gradient \( \nabla C(\theta_t) \).</li>
|
|
<li> Update first moment \( m_t \) and second moment \( v_t \) (exponential moving averages).</li>
|
|
<li> Bias-correct: \( \hat{m}_t = m_t/(1-\beta_1^t) \), \( \; \hat{v}_t = v_t/(1-\beta_2^t) \).</li>
|
|
<li> Compute step: \( \Delta \theta_t = \frac{\hat{m}_t}{\sqrt{\hat{v}_t} + \epsilon} \).</li>
|
|
<li> Update parameters: \( \theta_{t+1} = \theta_t - \alpha\, \Delta \theta_t \).</li>
|
|
</ol>
|
|
<p>This is the Adam update rule as given in the original paper.</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="adam-vs-adagrad-and-rmsprop">Adam vs. AdaGrad and RMSProp </h2>
|
|
|
|
<ol>
|
|
<li> AdaGrad: Uses per-coordinate scaling like Adam, but no momentum. Tends to slow down too much due to cumulative history (no forgetting)</li>
|
|
<li> RMSProp: Uses moving average of squared gradients (like Adam’s \( v_t \)) to maintain adaptive learning rates, but does not include momentum or bias-correction.</li>
|
|
<li> Adam: Effectively RMSProp + Momentum + Bias-correction</li>
|
|
<ul>
|
|
<li> Momentum (\( m_t \)) provides acceleration and smoother convergence.</li>
|
|
<li> Adaptive \( v_t \) scaling moderates the step size per dimension.</li>
|
|
<li> Bias correction (absent in AdaGrad/RMSProp) ensures robust estimates early on.</li>
|
|
</ul>
|
|
</ol>
|
|
<p>In practice, Adam often yields faster convergence and better tuning stability than RMSProp or AdaGrad alone</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="adaptivity-across-dimensions">Adaptivity Across Dimensions </h2>
|
|
|
|
<ol>
|
|
<li> Adam adapts the step size \emph{per coordinate}: parameters with larger gradient variance get smaller effective steps, those with smaller or sparse gradients get larger steps.</li>
|
|
<li> This per-dimension adaptivity is inherited from AdaGrad/RMSProp and helps handle ill-conditioned or sparse problems.</li>
|
|
<li> Meanwhile, momentum (first moment) allows Adam to continue making progress even if gradients become small or noisy, by leveraging accumulated direction.</li>
|
|
</ol>
|
|
<h2 id="adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html">ADAM algorithm, taken from <a href="https://www.deeplearningbook.org/contents/optimization.html" target="_blank">Goodfellow et al</a> </h2>
|
|
|
|
<br/><br/>
|
|
<center>
|
|
<p><img src="figures/adam.png" width="600" align="bottom"></p>
|
|
</center>
|
|
<br/><br/>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="algorithms-and-codes-for-adagrad-rmsprop-and-adam">Algorithms and codes for Adagrad, RMSprop and Adam </h2>
|
|
|
|
<p>The algorithms we have implemented are well described in the text by <a href="https://www.deeplearningbook.org/contents/optimization.html" target="_blank">Goodfellow, Bengio and Courville, chapter 8</a>.</p>
|
|
|
|
<p>The codes which implement these algorithms are discussed below here.</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="practical-tips">Practical tips </h2>
|
|
|
|
<ul>
|
|
<li> <b>Randomize the data when making mini-batches</b>. It is always important to randomly shuffle the data when forming mini-batches. Otherwise, the gradient descent method can fit spurious correlations resulting from the order in which data is presented.</li>
|
|
<li> <b>Transform your inputs</b>. Learning becomes difficult when our landscape has a mixture of steep and flat directions. One simple trick for minimizing these situations is to standardize the data by subtracting the mean and normalizing the variance of input variables. Whenever possible, also decorrelate the inputs. To understand why this is helpful, consider the case of linear regression. It is easy to show that for the squared error cost function, the Hessian of the cost function is just the correlation matrix between the inputs. Thus, by standardizing the inputs, we are ensuring that the landscape looks homogeneous in all directions in parameter space. Since most deep networks can be viewed as linear transformations followed by a non-linearity at each layer, we expect this intuition to hold beyond the linear case.</li>
|
|
<li> <b>Monitor the out-of-sample performance.</b> Always monitor the performance of your model on a validation set (a small portion of the training data that is held out of the training process to serve as a proxy for the test set. If the validation error starts increasing, then the model is beginning to overfit. Terminate the learning process. This <em>early stopping</em> significantly improves performance in many settings.</li>
|
|
<li> <b>Adaptive optimization methods don't always have good generalization.</b> Recent studies have shown that adaptive methods such as ADAM, RMSPorp, and AdaGrad tend to have poor generalization compared to SGD or SGD with momentum, particularly in the high-dimensional limit (i.e. the number of parameters exceeds the number of data points). Although it is not clear at this stage why these methods perform so well in training deep neural networks, simpler procedures like properly-tuned SGD may work as well or better in these applications.</li>
|
|
</ul>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="sneaking-in-automatic-differentiation-using-autograd">Sneaking in automatic differentiation using Autograd </h2>
|
|
|
|
<p>In the examples here we take the liberty of sneaking in automatic
|
|
differentiation (without having discussed the mathematics). In
|
|
project 1 you will write the gradients as discussed above, that is
|
|
hard-coding the gradients. By introducing automatic differentiation
|
|
via the library <b>autograd</b>, which is now replaced by <b>JAX</b>, we have
|
|
more flexibility in setting up alternative cost functions.
|
|
</p>
|
|
|
|
<p>The
|
|
first example shows results with ordinary leats squares.
|
|
</p>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients for OLS</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(theta):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> (<span style="color: #B452CD">1.0</span>/n)*np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
|
|
|
|
n = <span style="color: #B452CD">100</span>
|
|
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
|
|
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
|
|
|
|
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
|
|
XT_X = X.T @ X
|
|
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Own inversion"</span>)
|
|
<span style="color: #658b00">print</span>(theta_linreg)
|
|
<span style="color: #228B22"># Hessian matrix</span>
|
|
H = (<span style="color: #B452CD">2.0</span>/n)* XT_X
|
|
EigValues, EigVectors = np.linalg.eig(H)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"Eigenvalues of Hessian Matrix:{</span>EigValues<span style="color: #CD5555">}"</span>)
|
|
|
|
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
|
|
eta = <span style="color: #B452CD">1.0</span>/np.max(EigValues)
|
|
Niterations = <span style="color: #B452CD">1000</span>
|
|
<span style="color: #228B22"># define the gradient</span>
|
|
training_gradient = grad(CostOLS)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
|
|
gradients = training_gradient(theta)
|
|
theta -= eta*gradients
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own gd"</span>)
|
|
<span style="color: #658b00">print</span>(theta)
|
|
|
|
xnew = np.array([[<span style="color: #B452CD">0</span>],[<span style="color: #B452CD">2</span>]])
|
|
Xnew = np.c_[np.ones((<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)), xnew]
|
|
ypredict = Xnew.dot(theta)
|
|
ypredict2 = Xnew.dot(theta_linreg)
|
|
|
|
plt.plot(xnew, ypredict, <span style="color: #CD5555">"r-"</span>)
|
|
plt.plot(xnew, ypredict2, <span style="color: #CD5555">"b-"</span>)
|
|
plt.plot(x, y ,<span style="color: #CD5555">'ro'</span>)
|
|
plt.axis([<span style="color: #B452CD">0</span>,<span style="color: #B452CD">2.0</span>,<span style="color: #B452CD">0</span>, <span style="color: #B452CD">15.0</span>])
|
|
plt.xlabel(<span style="color: #CD5555">r'$x$'</span>)
|
|
plt.ylabel(<span style="color: #CD5555">r'$y$'</span>)
|
|
plt.title(<span style="color: #CD5555">r'Random numbers '</span>)
|
|
plt.show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="same-code-but-now-with-momentum-gradient-descent">Same code but now with momentum gradient descent </h2>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients for OLS</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(theta):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> (<span style="color: #B452CD">1.0</span>/n)*np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
|
|
|
|
n = <span style="color: #B452CD">100</span>
|
|
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
|
|
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x<span style="color: #228B22">#+np.random.randn(n,1)</span>
|
|
|
|
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
|
|
XT_X = X.T @ X
|
|
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Own inversion"</span>)
|
|
<span style="color: #658b00">print</span>(theta_linreg)
|
|
<span style="color: #228B22"># Hessian matrix</span>
|
|
H = (<span style="color: #B452CD">2.0</span>/n)* XT_X
|
|
EigValues, EigVectors = np.linalg.eig(H)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"Eigenvalues of Hessian Matrix:{</span>EigValues<span style="color: #CD5555">}"</span>)
|
|
|
|
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
|
|
eta = <span style="color: #B452CD">1.0</span>/np.max(EigValues)
|
|
Niterations = <span style="color: #B452CD">30</span>
|
|
|
|
<span style="color: #228B22"># define the gradient</span>
|
|
training_gradient = grad(CostOLS)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
|
|
gradients = training_gradient(theta)
|
|
theta -= eta*gradients
|
|
<span style="color: #658b00">print</span>(<span style="color: #658b00">iter</span>,gradients[<span style="color: #B452CD">0</span>],gradients[<span style="color: #B452CD">1</span>])
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own gd"</span>)
|
|
<span style="color: #658b00">print</span>(theta)
|
|
|
|
<span style="color: #228B22"># Now improve with momentum gradient descent</span>
|
|
change = <span style="color: #B452CD">0.0</span>
|
|
delta_momentum = <span style="color: #B452CD">0.3</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
|
|
<span style="color: #228B22"># calculate gradient</span>
|
|
gradients = training_gradient(theta)
|
|
<span style="color: #228B22"># calculate update</span>
|
|
new_change = eta*gradients+delta_momentum*change
|
|
<span style="color: #228B22"># take a step</span>
|
|
theta -= new_change
|
|
<span style="color: #228B22"># save the change</span>
|
|
change = new_change
|
|
<span style="color: #658b00">print</span>(<span style="color: #658b00">iter</span>,gradients[<span style="color: #B452CD">0</span>],gradients[<span style="color: #B452CD">1</span>])
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own gd wth momentum"</span>)
|
|
<span style="color: #658b00">print</span>(theta)
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="including-stochastic-gradient-descent-with-autograd">Including Stochastic Gradient Descent with Autograd </h2>
|
|
|
|
<p>In this code we include the stochastic gradient descent approach
|
|
discussed above. Note here that we specify which argument we are
|
|
taking the derivative with respect to when using <b>autograd</b>.
|
|
</p>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using SGD</span>
|
|
<span style="color: #228B22"># OLS example</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
|
|
|
|
<span style="color: #228B22"># Note change from previous example</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
|
|
|
|
n = <span style="color: #B452CD">100</span>
|
|
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
|
|
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
|
|
|
|
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
|
|
XT_X = X.T @ X
|
|
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Own inversion"</span>)
|
|
<span style="color: #658b00">print</span>(theta_linreg)
|
|
<span style="color: #228B22"># Hessian matrix</span>
|
|
H = (<span style="color: #B452CD">2.0</span>/n)* XT_X
|
|
EigValues, EigVectors = np.linalg.eig(H)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"Eigenvalues of Hessian Matrix:{</span>EigValues<span style="color: #CD5555">}"</span>)
|
|
|
|
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
|
|
eta = <span style="color: #B452CD">1.0</span>/np.max(EigValues)
|
|
Niterations = <span style="color: #B452CD">1000</span>
|
|
|
|
<span style="color: #228B22"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
|
|
training_gradient = grad(CostOLS,<span style="color: #B452CD">2</span>)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
|
|
gradients = (<span style="color: #B452CD">1.0</span>/n)*training_gradient(y, X, theta)
|
|
theta -= eta*gradients
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own gd"</span>)
|
|
<span style="color: #658b00">print</span>(theta)
|
|
|
|
xnew = np.array([[<span style="color: #B452CD">0</span>],[<span style="color: #B452CD">2</span>]])
|
|
Xnew = np.c_[np.ones((<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)), xnew]
|
|
ypredict = Xnew.dot(theta)
|
|
ypredict2 = Xnew.dot(theta_linreg)
|
|
|
|
plt.plot(xnew, ypredict, <span style="color: #CD5555">"r-"</span>)
|
|
plt.plot(xnew, ypredict2, <span style="color: #CD5555">"b-"</span>)
|
|
plt.plot(x, y ,<span style="color: #CD5555">'ro'</span>)
|
|
plt.axis([<span style="color: #B452CD">0</span>,<span style="color: #B452CD">2.0</span>,<span style="color: #B452CD">0</span>, <span style="color: #B452CD">15.0</span>])
|
|
plt.xlabel(<span style="color: #CD5555">r'$x$'</span>)
|
|
plt.ylabel(<span style="color: #CD5555">r'$y$'</span>)
|
|
plt.title(<span style="color: #CD5555">r'Random numbers '</span>)
|
|
plt.show()
|
|
|
|
n_epochs = <span style="color: #B452CD">50</span>
|
|
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
|
|
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
|
|
t0, t1 = <span style="color: #B452CD">5</span>, <span style="color: #B452CD">50</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">learning_schedule</span>(t):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> t0/(t+t1)
|
|
|
|
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
|
|
<span style="color: #228B22"># Can you figure out a better way of setting up the contributions to each batch?</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
|
|
random_index = M*np.random.randint(m)
|
|
xi = X[random_index:random_index+M]
|
|
yi = y[random_index:random_index+M]
|
|
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
|
|
eta = learning_schedule(epoch*m+i)
|
|
theta = theta - eta*gradients
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own sdg"</span>)
|
|
<span style="color: #658b00">print</span>(theta)
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="same-code-but-now-with-momentum-gradient-descent">Same code but now with momentum gradient descent </h2>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using SGD</span>
|
|
<span style="color: #228B22"># OLS example</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
|
|
|
|
<span style="color: #228B22"># Note change from previous example</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
|
|
|
|
n = <span style="color: #B452CD">100</span>
|
|
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
|
|
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
|
|
|
|
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
|
|
XT_X = X.T @ X
|
|
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Own inversion"</span>)
|
|
<span style="color: #658b00">print</span>(theta_linreg)
|
|
<span style="color: #228B22"># Hessian matrix</span>
|
|
H = (<span style="color: #B452CD">2.0</span>/n)* XT_X
|
|
EigValues, EigVectors = np.linalg.eig(H)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"Eigenvalues of Hessian Matrix:{</span>EigValues<span style="color: #CD5555">}"</span>)
|
|
|
|
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
|
|
eta = <span style="color: #B452CD">1.0</span>/np.max(EigValues)
|
|
Niterations = <span style="color: #B452CD">100</span>
|
|
|
|
<span style="color: #228B22"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
|
|
training_gradient = grad(CostOLS,<span style="color: #B452CD">2</span>)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
|
|
gradients = (<span style="color: #B452CD">1.0</span>/n)*training_gradient(y, X, theta)
|
|
theta -= eta*gradients
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own gd"</span>)
|
|
<span style="color: #658b00">print</span>(theta)
|
|
|
|
|
|
n_epochs = <span style="color: #B452CD">50</span>
|
|
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
|
|
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
|
|
t0, t1 = <span style="color: #B452CD">5</span>, <span style="color: #B452CD">50</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">learning_schedule</span>(t):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> t0/(t+t1)
|
|
|
|
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
|
|
|
|
change = <span style="color: #B452CD">0.0</span>
|
|
delta_momentum = <span style="color: #B452CD">0.3</span>
|
|
|
|
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
|
|
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
|
|
random_index = M*np.random.randint(m)
|
|
xi = X[random_index:random_index+M]
|
|
yi = y[random_index:random_index+M]
|
|
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
|
|
eta = learning_schedule(epoch*m+i)
|
|
<span style="color: #228B22"># calculate update</span>
|
|
new_change = eta*gradients+delta_momentum*change
|
|
<span style="color: #228B22"># take a step</span>
|
|
theta -= new_change
|
|
<span style="color: #228B22"># save the change</span>
|
|
change = new_change
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own sdg with momentum"</span>)
|
|
<span style="color: #658b00">print</span>(theta)
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="but-none-of-these-can-compete-with-newton-s-method">But none of these can compete with Newton's method </h2>
|
|
|
|
<p>Note that we here have introduced automatic differentiation</p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #228B22"># Using Newton's method</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(theta):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> (<span style="color: #B452CD">1.0</span>/n)*np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
|
|
|
|
n = <span style="color: #B452CD">100</span>
|
|
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
|
|
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+<span style="color: #B452CD">5</span>*x*x
|
|
|
|
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x, x*x]
|
|
XT_X = X.T @ X
|
|
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Own inversion"</span>)
|
|
<span style="color: #658b00">print</span>(theta_linreg)
|
|
<span style="color: #228B22"># Hessian matrix</span>
|
|
H = (<span style="color: #B452CD">2.0</span>/n)* XT_X
|
|
<span style="color: #228B22"># Note that here the Hessian does not depend on the parameters theta</span>
|
|
invH = np.linalg.pinv(H)
|
|
theta = np.random.randn(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">1</span>)
|
|
Niterations = <span style="color: #B452CD">5</span>
|
|
<span style="color: #228B22"># define the gradient</span>
|
|
training_gradient = grad(CostOLS)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
|
|
gradients = training_gradient(theta)
|
|
theta -= invH @ gradients
|
|
<span style="color: #658b00">print</span>(<span style="color: #658b00">iter</span>,gradients[<span style="color: #B452CD">0</span>],gradients[<span style="color: #B452CD">1</span>])
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own Newton code"</span>)
|
|
<span style="color: #658b00">print</span>(theta)
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="similar-second-order-function-now-problem-but-now-with-adagrad">Similar (second order function now) problem but now with AdaGrad </h2>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent</span>
|
|
<span style="color: #228B22"># OLS example</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
|
|
|
|
<span style="color: #228B22"># Note change from previous example</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
|
|
|
|
n = <span style="color: #B452CD">1000</span>
|
|
x = np.random.rand(n,<span style="color: #B452CD">1</span>)
|
|
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x
|
|
|
|
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x, x*x]
|
|
XT_X = X.T @ X
|
|
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Own inversion"</span>)
|
|
<span style="color: #658b00">print</span>(theta_linreg)
|
|
|
|
|
|
<span style="color: #228B22"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
|
|
training_gradient = grad(CostOLS,<span style="color: #B452CD">2</span>)
|
|
<span style="color: #228B22"># Define parameters for Stochastic Gradient Descent</span>
|
|
n_epochs = <span style="color: #B452CD">50</span>
|
|
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
|
|
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
|
|
<span style="color: #228B22"># Guess for unknown parameters theta</span>
|
|
theta = np.random.randn(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">1</span>)
|
|
|
|
<span style="color: #228B22"># Value for learning rate</span>
|
|
eta = <span style="color: #B452CD">0.01</span>
|
|
<span style="color: #228B22"># Including AdaGrad parameter to avoid possible division by zero</span>
|
|
delta = <span style="color: #B452CD">1e-8</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
|
|
Giter = <span style="color: #B452CD">0.0</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
|
|
random_index = M*np.random.randint(m)
|
|
xi = X[random_index:random_index+M]
|
|
yi = y[random_index:random_index+M]
|
|
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
|
|
Giter += gradients*gradients
|
|
update = gradients*eta/(delta+np.sqrt(Giter))
|
|
theta -= update
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own AdaGrad"</span>)
|
|
<span style="color: #658b00">print</span>(theta)
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<p>Running this code we note an almost perfect agreement with the results from matrix inversion.</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent">RMSprop for adaptive learning rate with Stochastic Gradient Descent </h2>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent</span>
|
|
<span style="color: #228B22"># OLS example</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
|
|
|
|
<span style="color: #228B22"># Note change from previous example</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
|
|
|
|
n = <span style="color: #B452CD">1000</span>
|
|
x = np.random.rand(n,<span style="color: #B452CD">1</span>)
|
|
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x<span style="color: #228B22"># +np.random.randn(n,1)</span>
|
|
|
|
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x, x*x]
|
|
XT_X = X.T @ X
|
|
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Own inversion"</span>)
|
|
<span style="color: #658b00">print</span>(theta_linreg)
|
|
|
|
|
|
<span style="color: #228B22"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
|
|
training_gradient = grad(CostOLS,<span style="color: #B452CD">2</span>)
|
|
<span style="color: #228B22"># Define parameters for Stochastic Gradient Descent</span>
|
|
n_epochs = <span style="color: #B452CD">50</span>
|
|
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
|
|
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
|
|
<span style="color: #228B22"># Guess for unknown parameters theta</span>
|
|
theta = np.random.randn(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">1</span>)
|
|
|
|
<span style="color: #228B22"># Value for learning rate</span>
|
|
eta = <span style="color: #B452CD">0.01</span>
|
|
<span style="color: #228B22"># Value for parameter rho</span>
|
|
rho = <span style="color: #B452CD">0.99</span>
|
|
<span style="color: #228B22"># Including AdaGrad parameter to avoid possible division by zero</span>
|
|
delta = <span style="color: #B452CD">1e-8</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
|
|
Giter = <span style="color: #B452CD">0.0</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
|
|
random_index = M*np.random.randint(m)
|
|
xi = X[random_index:random_index+M]
|
|
yi = y[random_index:random_index+M]
|
|
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
|
|
<span style="color: #228B22"># Accumulated gradient</span>
|
|
<span style="color: #228B22"># Scaling with rho the new and the previous results</span>
|
|
Giter = (rho*Giter+(<span style="color: #B452CD">1</span>-rho)*gradients*gradients)
|
|
<span style="color: #228B22"># Taking the diagonal only and inverting</span>
|
|
update = gradients*eta/(delta+np.sqrt(Giter))
|
|
<span style="color: #228B22"># Hadamard product</span>
|
|
theta -= update
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own RMSprop"</span>)
|
|
<span style="color: #658b00">print</span>(theta)
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf">And finally <a href="https://arxiv.org/pdf/1412.6980.pdf" target="_blank">ADAM</a> </h2>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent</span>
|
|
<span style="color: #228B22"># OLS example</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
|
|
|
|
<span style="color: #228B22"># Note change from previous example</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
|
|
|
|
n = <span style="color: #B452CD">1000</span>
|
|
x = np.random.rand(n,<span style="color: #B452CD">1</span>)
|
|
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x<span style="color: #228B22"># +np.random.randn(n,1)</span>
|
|
|
|
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x, x*x]
|
|
XT_X = X.T @ X
|
|
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Own inversion"</span>)
|
|
<span style="color: #658b00">print</span>(theta_linreg)
|
|
|
|
|
|
<span style="color: #228B22"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
|
|
training_gradient = grad(CostOLS,<span style="color: #B452CD">2</span>)
|
|
<span style="color: #228B22"># Define parameters for Stochastic Gradient Descent</span>
|
|
n_epochs = <span style="color: #B452CD">50</span>
|
|
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
|
|
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
|
|
<span style="color: #228B22"># Guess for unknown parameters theta</span>
|
|
theta = np.random.randn(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">1</span>)
|
|
|
|
<span style="color: #228B22"># Value for learning rate</span>
|
|
eta = <span style="color: #B452CD">0.01</span>
|
|
<span style="color: #228B22"># Value for parameters theta1 and theta2, see https://arxiv.org/abs/1412.6980</span>
|
|
theta1 = <span style="color: #B452CD">0.9</span>
|
|
theta2 = <span style="color: #B452CD">0.999</span>
|
|
<span style="color: #228B22"># Including AdaGrad parameter to avoid possible division by zero</span>
|
|
delta = <span style="color: #B452CD">1e-7</span>
|
|
<span style="color: #658b00">iter</span> = <span style="color: #B452CD">0</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
|
|
first_moment = <span style="color: #B452CD">0.0</span>
|
|
second_moment = <span style="color: #B452CD">0.0</span>
|
|
<span style="color: #658b00">iter</span> += <span style="color: #B452CD">1</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
|
|
random_index = M*np.random.randint(m)
|
|
xi = X[random_index:random_index+M]
|
|
yi = y[random_index:random_index+M]
|
|
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
|
|
<span style="color: #228B22"># Computing moments first</span>
|
|
first_moment = theta1*first_moment + (<span style="color: #B452CD">1</span>-theta1)*gradients
|
|
second_moment = theta2*second_moment+(<span style="color: #B452CD">1</span>-theta2)*gradients*gradients
|
|
first_term = first_moment/(<span style="color: #B452CD">1.0</span>-theta1**<span style="color: #658b00">iter</span>)
|
|
second_term = second_moment/(<span style="color: #B452CD">1.0</span>-theta2**<span style="color: #658b00">iter</span>)
|
|
<span style="color: #228B22"># Scaling with rho the new and the previous results</span>
|
|
update = eta*first_term/(np.sqrt(second_term)+delta)
|
|
theta -= update
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own ADAM"</span>)
|
|
<span style="color: #658b00">print</span>(theta)
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="material-for-the-lab-sessions">Material for the lab sessions </h2>
|
|
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
<ol>
|
|
<li> Exercise set for week 37 and reminder on scaling (from lab sessions of week 35)</li>
|
|
<li> Work on project 1
|
|
<!-- * <a href="https://youtu.be/bK4AEcTu-oM" target="_blank">Video of exercise sessions week 37</a> --></li>
|
|
</ol>
|
|
<p>For more discussions of Ridge regression and calculation of averages, <a href="https://arxiv.org/abs/1509.09169" target="_blank">Wessel van Wieringen's</a> article is highly recommended.</p>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="reminder-on-different-scaling-methods">Reminder on different scaling methods </h2>
|
|
|
|
<p>Before fitting a regression model, it is good practice to normalize or
|
|
standardize the features. This ensures all features are on a
|
|
comparable scale, which is especially important when using
|
|
regularization. In the exercises this week we will perform standardization, scaling each
|
|
feature to have mean 0 and standard deviation 1.
|
|
</p>
|
|
|
|
<p>Here we compute the mean and standard deviation of each column (feature) in our design/feature matrix \( \boldsymbol{X} \).
|
|
Then we subtract the mean and divide by the standard deviation for each feature.
|
|
</p>
|
|
|
|
<p>In the example here we
|
|
we will also center the target \( \boldsymbol{y} \) to mean \( 0 \). Centering \( \boldsymbol{y} \)
|
|
(and each feature) means the model does not require a separate intercept
|
|
term, the data is shifted such that the intercept is effectively 0
|
|
. (In practice, one could include an intercept in the model and not
|
|
penalize it, but here we simplify by centering.)
|
|
Choose \( n=100 \) data points and set up $\boldsymbol{x}, \( \boldsymbol{y} \) and the design matrix \( \boldsymbol{X} \).
|
|
</p>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #228B22"># Standardize features (zero mean, unit variance for each feature)</span>
|
|
X_mean = X.mean(axis=<span style="color: #B452CD">0</span>)
|
|
X_std = X.std(axis=<span style="color: #B452CD">0</span>)
|
|
X_std[X_std == <span style="color: #B452CD">0</span>] = <span style="color: #B452CD">1</span> <span style="color: #228B22"># safeguard to avoid division by zero for constant features</span>
|
|
X_norm = (X - X_mean) / X_std
|
|
|
|
<span style="color: #228B22"># Center the target to zero mean (optional, to simplify intercept handling)</span>
|
|
y_mean = <span style="color: #a61717; background-color: #e3d2d2">?</span>
|
|
y_centered = <span style="color: #a61717; background-color: #e3d2d2">?</span>
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<p>Do we need to center the values of \( y \)?</p>
|
|
|
|
<p>After this preprocessing, each column of \( \boldsymbol{X}_{\mathrm{norm}} \) has mean zero and standard deviation \( 1 \)
|
|
and \( \boldsymbol{y}_{\mathrm{centered}} \) has mean 0. This can make the optimization landscape
|
|
nicer and ensures the regularization penalty \( \lambda \sum_j
|
|
\theta_j^2 \) in Ridge regression treats each coefficient fairly (since features are on the
|
|
same scale).
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="functionality-in-scikit-learn">Functionality in Scikit-Learn </h2>
|
|
|
|
<p><b>Scikit-Learn</b> has several functions which allow us to rescale the
|
|
data, normally resulting in much better results in terms of various
|
|
accuracy scores. The <b>StandardScaler</b> function in <b>Scikit-Learn</b>
|
|
ensures that for each feature/predictor we study the mean value is
|
|
zero and the variance is one (every column in the design/feature
|
|
matrix). This scaling has the drawback that it does not ensure that
|
|
we have a particular maximum or minimum in our data set. Another
|
|
function included in <b>Scikit-Learn</b> is the <b>MinMaxScaler</b> which
|
|
ensures that all features are exactly between \( 0 \) and \( 1 \). The
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="more-preprocessing">More preprocessing </h2>
|
|
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
<p>The <b>Normalizer</b> scales each data
|
|
point such that the feature vector has a euclidean length of one. In other words, it
|
|
projects a data point on the circle (or sphere in the case of higher dimensions) with a
|
|
radius of 1. This means every data point is scaled by a different number (by the
|
|
inverse of it’s length).
|
|
This normalization is often used when only the direction (or angle) of the data matters,
|
|
not the length of the feature vector.
|
|
</p>
|
|
|
|
<p>The <b>RobustScaler</b> works similarly to the StandardScaler in that it
|
|
ensures statistical properties for each feature that guarantee that
|
|
they are on the same scale. However, the RobustScaler uses the median
|
|
and quartiles, instead of mean and variance. This makes the
|
|
RobustScaler ignore data points that are very different from the rest
|
|
(like measurement errors). These odd data points are also called
|
|
outliers, and might often lead to trouble for other scaling
|
|
techniques.
|
|
</p>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="frequently-used-scaling-functions">Frequently used scaling functions </h2>
|
|
|
|
<p>Many features are often scaled using standardization to improve performance. In <b>Scikit-Learn</b> this is given by the <b>StandardScaler</b> function as discussed above. It is easy however to write your own.
|
|
Mathematically, this involves subtracting the mean and divide by the standard deviation over the data set, for each feature:
|
|
</p>
|
|
|
|
$$
|
|
x_j^{(i)} \rightarrow \frac{x_j^{(i)} - \overline{x}_j}{\sigma(x_j)},
|
|
$$
|
|
|
|
<p>where \( \overline{x}_j \) and \( \sigma(x_j) \) are the mean and standard deviation, respectively, of the feature \( x_j \).
|
|
This ensures that each feature has zero mean and unit standard deviation. For data sets where we do not have the standard deviation or don't wish to calculate it, it is then common to simply set it to one.
|
|
</p>
|
|
|
|
<p>Keep in mind that when you transform your data set before training a model, the same transformation needs to be done
|
|
on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as
|
|
</p>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #CD5555">"""</span>
|
|
<span style="color: #CD5555">#Model training, we compute the mean value of y and X</span>
|
|
<span style="color: #CD5555">y_train_mean = np.mean(y_train)</span>
|
|
<span style="color: #CD5555">X_train_mean = np.mean(X_train,axis=0)</span>
|
|
<span style="color: #CD5555">X_train = X_train - X_train_mean</span>
|
|
<span style="color: #CD5555">y_train = y_train - y_train_mean</span>
|
|
|
|
<span style="color: #CD5555"># The we fit our model with the training data</span>
|
|
<span style="color: #CD5555">trained_model = some_model.fit(X_train,y_train)</span>
|
|
|
|
|
|
<span style="color: #CD5555">#Model prediction, we need also to transform our data set used for the prediction.</span>
|
|
<span style="color: #CD5555">X_test = X_test - X_train_mean #Use mean from training data</span>
|
|
<span style="color: #CD5555">y_pred = trained_model(X_test)</span>
|
|
<span style="color: #CD5555">y_pred = y_pred + y_train_mean</span>
|
|
<span style="color: #CD5555">"""</span>
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<p>Let us try to understand what this may imply mathematically when we
|
|
subtract the mean values, also known as <em>zero centering</em>. For
|
|
simplicity, we will focus on ordinary regression, as done in the above example.
|
|
</p>
|
|
|
|
<p>The cost/loss function for regression is</p>
|
|
$$
|
|
C(\theta_0, \theta_1, ... , \theta_{p-1}) = \frac{1}{n}\sum_{i=0}^{n} \left(y_i - \theta_0 - \sum_{j=1}^{p-1} X_{ij}\theta_j\right)^2,.
|
|
$$
|
|
|
|
<p>Recall also that we use the squared value. This expression can lead to an
|
|
increased penalty for higher differences between predicted and
|
|
output/target values.
|
|
</p>
|
|
|
|
<p>What we have done is to single out the \( \theta_0 \) term in the
|
|
definition of the mean squared error (MSE). The design matrix \( X \)
|
|
does in this case not contain any intercept column. When we take the
|
|
derivative with respect to \( \theta_0 \), we want the derivative to obey
|
|
</p>
|
|
|
|
$$
|
|
\frac{\partial C}{\partial \theta_j} = 0,
|
|
$$
|
|
|
|
<p>for all \( j \). For \( \theta_0 \) we have</p>
|
|
|
|
$$
|
|
\frac{\partial C}{\partial \theta_0} = -\frac{2}{n}\sum_{i=0}^{n-1} \left(y_i - \theta_0 - \sum_{j=1}^{p-1} X_{ij} \theta_j\right).
|
|
$$
|
|
|
|
<p>Multiplying away the constant \( 2/n \), we obtain</p>
|
|
$$
|
|
\sum_{i=0}^{n-1} \theta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} \sum_{j=1}^{p-1} X_{ij} \theta_j.
|
|
$$
|
|
|
|
<p>Let us specialize first to the case where we have only two parameters \( \theta_0 \) and \( \theta_1 \).
|
|
Our result for \( \theta_0 \) simplifies then to
|
|
</p>
|
|
$$
|
|
n\theta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} X_{i1} \theta_1.
|
|
$$
|
|
|
|
<p>We obtain then</p>
|
|
$$
|
|
\theta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \theta_1\frac{1}{n}\sum_{i=0}^{n-1} X_{i1}.
|
|
$$
|
|
|
|
<p>If we define</p>
|
|
$$
|
|
\mu_{\boldsymbol{x}_1}=\frac{1}{n}\sum_{i=0}^{n-1} X_{i1},
|
|
$$
|
|
|
|
<p>and the mean value of the outputs as</p>
|
|
$$
|
|
\mu_y=\frac{1}{n}\sum_{i=0}^{n-1}y_i,
|
|
$$
|
|
|
|
<p>we have</p>
|
|
$$
|
|
\theta_0 = \mu_y - \theta_1\mu_{\boldsymbol{x}_1}.
|
|
$$
|
|
|
|
<p>In the general case with more parameters than \( \theta_0 \) and \( \theta_1 \), we have</p>
|
|
$$
|
|
\theta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \frac{1}{n}\sum_{i=0}^{n-1}\sum_{j=1}^{p-1} X_{ij}\theta_j.
|
|
$$
|
|
|
|
<p>We can rewrite the latter equation as</p>
|
|
$$
|
|
\theta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \sum_{j=1}^{p-1} \mu_{\boldsymbol{x}_j}\theta_j,
|
|
$$
|
|
|
|
<p>where we have defined</p>
|
|
$$
|
|
\mu_{\boldsymbol{x}_j}=\frac{1}{n}\sum_{i=0}^{n-1} X_{ij},
|
|
$$
|
|
|
|
<p>the mean value for all elements of the column vector \( \boldsymbol{x}_j \).</p>
|
|
|
|
<p>Replacing \( y_i \) with \( y_i - y_i - \overline{\boldsymbol{y}} \) and centering also our design matrix results in a cost function (in vector-matrix disguise)</p>
|
|
$$
|
|
C(\boldsymbol{\theta}) = (\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\theta})^T(\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\theta}).
|
|
$$
|
|
|
|
<p>If we minimize with respect to \( \boldsymbol{\theta} \) we have then</p>
|
|
|
|
$$
|
|
\hat{\boldsymbol{\theta}} = (\tilde{X}^T\tilde{X})^{-1}\tilde{X}^T\boldsymbol{\tilde{y}},
|
|
$$
|
|
|
|
<p>where \( \boldsymbol{\tilde{y}} = \boldsymbol{y} - \overline{\boldsymbol{y}} \)
|
|
and \( \tilde{X}_{ij} = X_{ij} - \frac{1}{n}\sum_{k=0}^{n-1}X_{kj} \).
|
|
</p>
|
|
|
|
<p>For Ridge regression we need to add \( \lambda \boldsymbol{\theta}^T\boldsymbol{\theta} \) to the cost function and get then</p>
|
|
$$
|
|
\hat{\boldsymbol{\theta}} = (\tilde{X}^T\tilde{X} + \lambda I)^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}.
|
|
$$
|
|
|
|
<p>What does this mean? And why do we insist on all this? Let us look at some examples.</p>
|
|
|
|
<p>This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (<em>code example thanks to Øyvind Sigmundson Schøyen</em>). Here our scaling of the data is done by subtracting the mean values only.
|
|
Note also that we do not split the data into training and test.
|
|
</p>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearRegression
|
|
|
|
|
|
np.random.seed(<span style="color: #B452CD">2021</span>)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">MSE</span>(y_data,y_model):
|
|
n = np.size(y_model)
|
|
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y_data-y_model)**<span style="color: #B452CD">2</span>)/n
|
|
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">fit_theta</span>(X, y):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> np.linalg.pinv(X.T @ X) @ X.T @ y
|
|
|
|
|
|
true_theta = [<span style="color: #B452CD">2</span>, <span style="color: #B452CD">0.5</span>, <span style="color: #B452CD">3.7</span>]
|
|
|
|
x = np.linspace(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">11</span>)
|
|
y = np.sum(
|
|
np.asarray([x ** p * b <span style="color: #8B008B; font-weight: bold">for</span> p, b <span style="color: #8B008B">in</span> <span style="color: #658b00">enumerate</span>(true_theta)]), axis=<span style="color: #B452CD">0</span>
|
|
) + <span style="color: #B452CD">0.1</span> * np.random.normal(size=<span style="color: #658b00">len</span>(x))
|
|
|
|
degree = <span style="color: #B452CD">3</span>
|
|
X = np.zeros((<span style="color: #658b00">len</span>(x), degree))
|
|
|
|
<span style="color: #228B22"># Include the intercept in the design matrix</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> p <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(degree):
|
|
X[:, p] = x ** p
|
|
|
|
theta = fit_theta(X, y)
|
|
|
|
<span style="color: #228B22"># Intercept is included in the design matrix</span>
|
|
skl = LinearRegression(fit_intercept=<span style="color: #8B008B; font-weight: bold">False</span>).fit(X, y)
|
|
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"True theta: {</span>true_theta<span style="color: #CD5555">}"</span>)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"Fitted theta: {</span>theta<span style="color: #CD5555">}"</span>)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"Sklearn fitted theta: {</span>skl.coef_<span style="color: #CD5555">}"</span>)
|
|
ypredictOwn = X @ theta
|
|
ypredictSKL = skl.predict(X)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"MSE with intercept column"</span>)
|
|
<span style="color: #658b00">print</span>(MSE(y,ypredictOwn))
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"MSE with intercept column from SKL"</span>)
|
|
<span style="color: #658b00">print</span>(MSE(y,ypredictSKL))
|
|
|
|
|
|
plt.figure()
|
|
plt.scatter(x, y, label=<span style="color: #CD5555">"Data"</span>)
|
|
plt.plot(x, X @ theta, label=<span style="color: #CD5555">"Fit"</span>)
|
|
plt.plot(x, skl.predict(X), label=<span style="color: #CD5555">"Sklearn (fit_intercept=False)"</span>)
|
|
|
|
|
|
<span style="color: #228B22"># Do not include the intercept in the design matrix</span>
|
|
X = np.zeros((<span style="color: #658b00">len</span>(x), degree - <span style="color: #B452CD">1</span>))
|
|
|
|
<span style="color: #8B008B; font-weight: bold">for</span> p <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(degree - <span style="color: #B452CD">1</span>):
|
|
X[:, p] = x ** (p + <span style="color: #B452CD">1</span>)
|
|
|
|
<span style="color: #228B22"># Intercept is not included in the design matrix</span>
|
|
skl = LinearRegression(fit_intercept=<span style="color: #8B008B; font-weight: bold">True</span>).fit(X, y)
|
|
|
|
<span style="color: #228B22"># Use centered values for X and y when computing coefficients</span>
|
|
y_offset = np.average(y, axis=<span style="color: #B452CD">0</span>)
|
|
X_offset = np.average(X, axis=<span style="color: #B452CD">0</span>)
|
|
|
|
theta = fit_theta(X - X_offset, y - y_offset)
|
|
intercept = np.mean(y_offset - X_offset @ theta)
|
|
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"Manual intercept: {</span>intercept<span style="color: #CD5555">}"</span>)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"Fitted theta (without intercept): {</span>theta<span style="color: #CD5555">}"</span>)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"Sklearn intercept: {</span>skl.intercept_<span style="color: #CD5555">}"</span>)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"Sklearn fitted theta (without intercept): {</span>skl.coef_<span style="color: #CD5555">}"</span>)
|
|
ypredictOwn = X @ theta
|
|
ypredictSKL = skl.predict(X)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"MSE with Manual intercept"</span>)
|
|
<span style="color: #658b00">print</span>(MSE(y,ypredictOwn+intercept))
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"MSE with Sklearn intercept"</span>)
|
|
<span style="color: #658b00">print</span>(MSE(y,ypredictSKL))
|
|
|
|
plt.plot(x, X @ theta + intercept, <span style="color: #CD5555">"--"</span>, label=<span style="color: #CD5555">"Fit (manual intercept)"</span>)
|
|
plt.plot(x, skl.predict(X), <span style="color: #CD5555">"--"</span>, label=<span style="color: #CD5555">"Sklearn (fit_intercept=True)"</span>)
|
|
plt.grid()
|
|
plt.legend()
|
|
|
|
plt.show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<p>The intercept is the value of our output/target variable
|
|
when all our features are zero and our function crosses the \( y \)-axis (for a one-dimensional case).
|
|
</p>
|
|
|
|
<p>Printing the MSE, we see first that both methods give the same MSE, as
|
|
they should. However, when we move to for example Ridge regression,
|
|
the way we treat the intercept may give a larger or smaller MSE,
|
|
meaning that the MSE can be penalized by the value of the
|
|
intercept. Not including the intercept in the fit, means that the
|
|
regularization term does not include \( \theta_0 \). For different values
|
|
of \( \lambda \), this may lead to different MSE values.
|
|
</p>
|
|
|
|
<p>To remind the reader, the regularization term, with the intercept in Ridge regression, is given by</p>
|
|
$$
|
|
\lambda \vert\vert \boldsymbol{\theta} \vert\vert_2^2 = \lambda \sum_{j=0}^{p-1}\theta_j^2,
|
|
$$
|
|
|
|
<p>but when we take out the intercept, this equation becomes</p>
|
|
$$
|
|
\lambda \vert\vert \boldsymbol{\theta} \vert\vert_2^2 = \lambda \sum_{j=1}^{p-1}\theta_j^2.
|
|
$$
|
|
|
|
<p>For Lasso regression we have</p>
|
|
$$
|
|
\lambda \vert\vert \boldsymbol{\theta} \vert\vert_1 = \lambda \sum_{j=1}^{p-1}\vert\theta_j\vert.
|
|
$$
|
|
|
|
<p>It means that, when scaling the design matrix and the outputs/targets,
|
|
by subtracting the mean values, we have an optimization problem which
|
|
is not penalized by the intercept. The MSE value can then be smaller
|
|
since it focuses only on the remaining quantities. If we however bring
|
|
back the intercept, we will get a MSE which then contains the
|
|
intercept.
|
|
</p>
|
|
|
|
<p>Armed with this wisdom, we attempt first to simply set the intercept equal to <b>False</b> in our implementation of Ridge regression for our well-known vanilla data set.</p>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn</span> <span style="color: #8B008B; font-weight: bold">import</span> linear_model
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">MSE</span>(y_data,y_model):
|
|
n = np.size(y_model)
|
|
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y_data-y_model)**<span style="color: #B452CD">2</span>)/n
|
|
|
|
|
|
<span style="color: #228B22"># A seed just to ensure that the random numbers are the same for every run.</span>
|
|
<span style="color: #228B22"># Useful for eventual debugging.</span>
|
|
np.random.seed(<span style="color: #B452CD">3155</span>)
|
|
|
|
n = <span style="color: #B452CD">100</span>
|
|
x = np.random.rand(n)
|
|
y = np.exp(-x**<span style="color: #B452CD">2</span>) + <span style="color: #B452CD">1.5</span> * np.exp(-(x-<span style="color: #B452CD">2</span>)**<span style="color: #B452CD">2</span>)
|
|
|
|
Maxpolydegree = <span style="color: #B452CD">20</span>
|
|
X = np.zeros((n,Maxpolydegree))
|
|
<span style="color: #228B22">#We include explicitely the intercept column</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> degree <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Maxpolydegree):
|
|
X[:,degree] = x**degree
|
|
<span style="color: #228B22"># We split the data in test and training data</span>
|
|
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=<span style="color: #B452CD">0.2</span>)
|
|
|
|
p = Maxpolydegree
|
|
I = np.eye(p,p)
|
|
<span style="color: #228B22"># Decide which values of lambda to use</span>
|
|
nlambdas = <span style="color: #B452CD">6</span>
|
|
MSEOwnRidgePredict = np.zeros(nlambdas)
|
|
MSERidgePredict = np.zeros(nlambdas)
|
|
lambdas = np.logspace(-<span style="color: #B452CD">4</span>, <span style="color: #B452CD">2</span>, nlambdas)
|
|
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(nlambdas):
|
|
lmb = lambdas[i]
|
|
OwnRidgeTheta = np.linalg.pinv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train
|
|
<span style="color: #228B22"># Note: we include the intercept column and no scaling</span>
|
|
RegRidge = linear_model.Ridge(lmb,fit_intercept=<span style="color: #8B008B; font-weight: bold">False</span>)
|
|
RegRidge.fit(X_train,y_train)
|
|
<span style="color: #228B22"># and then make the prediction</span>
|
|
ytildeOwnRidge = X_train @ OwnRidgeTheta
|
|
ypredictOwnRidge = X_test @ OwnRidgeTheta
|
|
ytildeRidge = RegRidge.predict(X_train)
|
|
ypredictRidge = RegRidge.predict(X_test)
|
|
MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)
|
|
MSERidgePredict[i] = MSE(y_test,ypredictRidge)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Theta values for own Ridge implementation"</span>)
|
|
<span style="color: #658b00">print</span>(OwnRidgeTheta)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Theta values for Scikit-Learn Ridge implementation"</span>)
|
|
<span style="color: #658b00">print</span>(RegRidge.coef_)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"MSE values for own Ridge implementation"</span>)
|
|
<span style="color: #658b00">print</span>(MSEOwnRidgePredict[i])
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"MSE values for Scikit-Learn Ridge implementation"</span>)
|
|
<span style="color: #658b00">print</span>(MSERidgePredict[i])
|
|
|
|
<span style="color: #228B22"># Now plot the results</span>
|
|
plt.figure()
|
|
plt.plot(np.log10(lambdas), MSEOwnRidgePredict, <span style="color: #CD5555">'r'</span>, label = <span style="color: #CD5555">'MSE own Ridge Test'</span>)
|
|
plt.plot(np.log10(lambdas), MSERidgePredict, <span style="color: #CD5555">'g'</span>, label = <span style="color: #CD5555">'MSE Ridge Test'</span>)
|
|
|
|
plt.xlabel(<span style="color: #CD5555">'log10(lambda)'</span>)
|
|
plt.ylabel(<span style="color: #CD5555">'MSE'</span>)
|
|
plt.legend()
|
|
plt.show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<p>The results here agree when we force <b>Scikit-Learn</b>'s Ridge function to include the first column in our design matrix.
|
|
We see that the results agree very well. Here we have thus explicitely included the intercept column in the design matrix.
|
|
What happens if we do not include the intercept in our fit?
|
|
Let us see how we can change this code by zero centering.
|
|
</p>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #eeeedd">
|
|
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn</span> <span style="color: #8B008B; font-weight: bold">import</span> linear_model
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> StandardScaler
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">MSE</span>(y_data,y_model):
|
|
n = np.size(y_model)
|
|
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y_data-y_model)**<span style="color: #B452CD">2</span>)/n
|
|
<span style="color: #228B22"># A seed just to ensure that the random numbers are the same for every run.</span>
|
|
<span style="color: #228B22"># Useful for eventual debugging.</span>
|
|
np.random.seed(<span style="color: #B452CD">315</span>)
|
|
|
|
n = <span style="color: #B452CD">100</span>
|
|
x = np.random.rand(n)
|
|
y = np.exp(-x**<span style="color: #B452CD">2</span>) + <span style="color: #B452CD">1.5</span> * np.exp(-(x-<span style="color: #B452CD">2</span>)**<span style="color: #B452CD">2</span>)
|
|
|
|
Maxpolydegree = <span style="color: #B452CD">20</span>
|
|
X = np.zeros((n,Maxpolydegree-<span style="color: #B452CD">1</span>))
|
|
|
|
<span style="color: #8B008B; font-weight: bold">for</span> degree <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #B452CD">1</span>,Maxpolydegree): <span style="color: #228B22">#No intercept column</span>
|
|
X[:,degree-<span style="color: #B452CD">1</span>] = x**(degree)
|
|
|
|
<span style="color: #228B22"># We split the data in test and training data</span>
|
|
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=<span style="color: #B452CD">0.2</span>)
|
|
|
|
<span style="color: #228B22">#For our own implementation, we will need to deal with the intercept by centering the design matrix and the target variable</span>
|
|
X_train_mean = np.mean(X_train,axis=<span style="color: #B452CD">0</span>)
|
|
<span style="color: #228B22">#Center by removing mean from each feature</span>
|
|
X_train_scaled = X_train - X_train_mean
|
|
X_test_scaled = X_test - X_train_mean
|
|
<span style="color: #228B22">#The model intercept (called y_scaler) is given by the mean of the target variable (IF X is centered)</span>
|
|
<span style="color: #228B22">#Remove the intercept from the training data.</span>
|
|
y_scaler = np.mean(y_train)
|
|
y_train_scaled = y_train - y_scaler
|
|
|
|
p = Maxpolydegree-<span style="color: #B452CD">1</span>
|
|
I = np.eye(p,p)
|
|
<span style="color: #228B22"># Decide which values of lambda to use</span>
|
|
nlambdas = <span style="color: #B452CD">6</span>
|
|
MSEOwnRidgePredict = np.zeros(nlambdas)
|
|
MSERidgePredict = np.zeros(nlambdas)
|
|
|
|
lambdas = np.logspace(-<span style="color: #B452CD">4</span>, <span style="color: #B452CD">2</span>, nlambdas)
|
|
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(nlambdas):
|
|
lmb = lambdas[i]
|
|
OwnRidgeTheta = np.linalg.pinv(X_train_scaled.T @ X_train_scaled+lmb*I) @ X_train_scaled.T @ (y_train_scaled)
|
|
intercept_ = y_scaler - X_train_mean<span style="color: #707a7c">@OwnRidgeTheta</span> <span style="color: #228B22">#The intercept can be shifted so the model can predict on uncentered data</span>
|
|
<span style="color: #228B22">#Add intercept to prediction</span>
|
|
ypredictOwnRidge = X_test_scaled @ OwnRidgeTheta + y_scaler
|
|
RegRidge = linear_model.Ridge(lmb)
|
|
RegRidge.fit(X_train,y_train)
|
|
ypredictRidge = RegRidge.predict(X_test)
|
|
MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)
|
|
MSERidgePredict[i] = MSE(y_test,ypredictRidge)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Theta values for own Ridge implementation"</span>)
|
|
<span style="color: #658b00">print</span>(OwnRidgeTheta) <span style="color: #228B22">#Intercept is given by mean of target variable</span>
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Theta values for Scikit-Learn Ridge implementation"</span>)
|
|
<span style="color: #658b00">print</span>(RegRidge.coef_)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">'Intercept from own implementation:'</span>)
|
|
<span style="color: #658b00">print</span>(intercept_)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">'Intercept from Scikit-Learn Ridge implementation'</span>)
|
|
<span style="color: #658b00">print</span>(RegRidge.intercept_)
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"MSE values for own Ridge implementation"</span>)
|
|
<span style="color: #658b00">print</span>(MSEOwnRidgePredict[i])
|
|
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"MSE values for Scikit-Learn Ridge implementation"</span>)
|
|
<span style="color: #658b00">print</span>(MSERidgePredict[i])
|
|
|
|
|
|
<span style="color: #228B22"># Now plot the results</span>
|
|
plt.figure()
|
|
plt.plot(np.log10(lambdas), MSEOwnRidgePredict, <span style="color: #CD5555">'b--'</span>, label = <span style="color: #CD5555">'MSE own Ridge Test'</span>)
|
|
plt.plot(np.log10(lambdas), MSERidgePredict, <span style="color: #CD5555">'g--'</span>, label = <span style="color: #CD5555">'MSE SL Ridge Test'</span>)
|
|
plt.xlabel(<span style="color: #CD5555">'log10(lambda)'</span>)
|
|
plt.ylabel(<span style="color: #CD5555">'MSE'</span>)
|
|
plt.legend()
|
|
plt.show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<p>We see here, when compared to the code which includes explicitely the
|
|
intercept column, that our MSE value is actually smaller. This is
|
|
because the regularization term does not include the intercept value
|
|
\( \theta_0 \) in the fitting. This applies to Lasso regularization as
|
|
well. It means that our optimization is now done only with the
|
|
centered matrix and/or vector that enter the fitting procedure.
|
|
</p>
|
|
|
|
<!-- ------------------- end of main content --------------- -->
|
|
<center style="font-size:80%">
|
|
<!-- copyright --> © 1999-2025, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|