Morten Hjorth-Jensen
1776 lines
144 KiB
HTML
1776 lines
144 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
<html>
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<title>Exercises week 41 — Applied Data Analysis and Machine Learning</title>
|
||
|
||
<link href="_static/css/theme.css" rel="stylesheet">
|
||
<link href="_static/css/index.ff1ffe594081f20da1ef19478df9384b.css" rel="stylesheet">
|
||
|
||
|
||
<link rel="stylesheet"
|
||
href="_static/vendor/fontawesome/5.13.0/css/all.min.css">
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin
|
||
href="_static/vendor/fontawesome/5.13.0/webfonts/fa-solid-900.woff2">
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin
|
||
href="_static/vendor/fontawesome/5.13.0/webfonts/fa-brands-400.woff2">
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-book-theme.css?digest=c3fdc42140077d1ad13ad2f1588a4309" />
|
||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/mystnb.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/panels-main.c949a650a448cc0ae9fd3441c0e17fb0.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/panels-variables.06eb56fa6e07937060861dad626602ad.css" />
|
||
|
||
<link rel="preload" as="script" href="_static/js/index.be7d3bbb2ef33a8344ce.js">
|
||
|
||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||
<script src="_static/jquery.js"></script>
|
||
<script src="_static/underscore.js"></script>
|
||
<script src="_static/doctools.js"></script>
|
||
<script src="_static/clipboard.min.js"></script>
|
||
<script src="_static/copybutton.js"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="_static/togglebutton.js"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown, .tag_hide_input div.cell_input, .tag_hide-input div.cell_input, .tag_hide_output div.cell_output, .tag_hide-output div.cell_output, .tag_hide_cell.cell, .tag_hide-cell.cell';</script>
|
||
<script src="_static/sphinx-book-theme.d59cb220de22ca1c485ebbdc042f0030.js"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"
|
||
const thebe_selector = ".thebe,.cell"
|
||
const thebe_selector_input = "pre"
|
||
const thebe_selector_output = ".output, .cell_output"
|
||
</script>
|
||
<script async="async" src="_static/sphinx-thebe.js"></script>
|
||
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
||
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="Week 41 Neural networks and constructing a neural network code" href="week41.html" />
|
||
<link rel="prev" title="Week 40: Gradient descent methods (continued) and start Neural networks" href="week40.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
<meta name="docsearch:language" content="None">
|
||
|
||
|
||
<!-- Google Analytics -->
|
||
|
||
</head>
|
||
<body data-spy="scroll" data-target="#bd-toc-nav" data-offset="80">
|
||
|
||
<div class="container-fluid" id="banner"></div>
|
||
|
||
|
||
|
||
<div class="container-xl">
|
||
<div class="row">
|
||
|
||
<div class="col-12 col-md-3 bd-sidebar site-navigation show" id="site-navigation">
|
||
|
||
<div class="navbar-brand-box">
|
||
<a class="navbar-brand text-wrap" href="index.html">
|
||
|
||
<!-- `logo` is deprecated in Sphinx 4.0, so remove this when we stop supporting 3 -->
|
||
|
||
|
||
|
||
<img src="_static/logo.png" class="logo" alt="logo">
|
||
|
||
|
||
<h1 class="site-logo" id="site-title">Applied Data Analysis and Machine Learning</h1>
|
||
|
||
</a>
|
||
</div><form class="bd-search d-flex align-items-center" action="search.html" method="get">
|
||
<i class="icon fas fa-search"></i>
|
||
<input type="search" class="form-control" name="q" id="search-input" placeholder="Search this book..." aria-label="Search this book..." autocomplete="off" >
|
||
</form><nav class="bd-links" id="bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item active">
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="intro.html">
|
||
Applied Data Analysis and Machine Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
About the course
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="schedule.html">
|
||
Teaching schedule with links to material
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="teachers.html">
|
||
Teachers and Grading
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="textbooks.html">
|
||
Textbooks
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Review of Statistics with Resampling Techniques and Linear Algebra
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="statistics.html">
|
||
1. Elements of Probability Theory and Statistical Data Analysis
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="linalg.html">
|
||
2. Linear Algebra, Handling of Arrays and more Python Features
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
From Regression to Support Vector Machines
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter1.html">
|
||
3. Linear Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter2.html">
|
||
4. Ridge and Lasso Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter3.html">
|
||
5. Resampling Methods
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter4.html">
|
||
6. Logistic Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapteroptimization.html">
|
||
7. Optimization, the central part of any Machine Learning algortithm
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter5.html">
|
||
8. Support Vector Machines, overarching aims
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Decision Trees, Ensemble Methods and Boosting
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter6.html">
|
||
9. Decision trees, overarching aims
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter7.html">
|
||
10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Dimensionality Reduction
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter8.html">
|
||
11. Basic ideas of the Principal Component Analysis (PCA)
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="clustering.html">
|
||
12. Clustering and Unsupervised Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Deep Learning Methods
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter9.html">
|
||
13. Neural networks
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter10.html">
|
||
14. Building a Feed Forward Neural Network
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter11.html">
|
||
15. Solving Differential Equations with Deep Learning
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter12.html">
|
||
16. Convolutional Neural Networks
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter13.html">
|
||
17. Recurrent neural networks: Overarching view
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Weekly material, notes and exercises
|
||
</span>
|
||
</p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek34.html">
|
||
Exercises week 34
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week34.html">
|
||
Week 34: Introduction to the course, Logistics and Practicalities
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek35.html">
|
||
Exercises week 35
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week35.html">
|
||
Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek36.html">
|
||
Exercises week 36
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week36.html">
|
||
Week 36: Linear Regression and Statistical interpretations
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek37.html">
|
||
Exercises week 37
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week37.html">
|
||
Week 37: Statistical interpretations and Resampling Methods
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek38.html">
|
||
Exercises week 38
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week38.html">
|
||
Week 38: Logistic Regression and Optimization
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek39.html">
|
||
Exercises week 39
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week39.html">
|
||
Week 39: Optimization and Gradient Methods
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week40.html">
|
||
Week 40: Gradient descent methods (continued) and start Neural networks
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1 current active">
|
||
<a class="current reference internal" href="#">
|
||
Exercises week 41
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week41.html">
|
||
Week 41 Neural networks and constructing a neural network code
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek42.html">
|
||
Exercises week 42
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week42.html">
|
||
Week 42 Constructing a Neural Network code with examples
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="additionweek42.html">
|
||
Exercises Week 42: Logistic Regression and Optimization, reminders from week 38 and week 40
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week43.html">
|
||
Week 43: Deep Learning: Constructing a Neural Network code and solving differential equations
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek43.html">
|
||
Exercises week 43
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Projects
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="project1.html">
|
||
Project 1 on Machine Learning, deadline October 7 (midnight), 2024
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="project2.html">
|
||
Project 2 on Machine Learning, deadline November 4 (Midnight)
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</div>
|
||
</nav> <!-- To handle the deprecated key -->
|
||
|
||
<div class="navbar_extra_footer">
|
||
Powered by <a href="https://jupyterbook.org">Jupyter Book</a>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<main class="col py-md-3 pl-md-4 bd-content overflow-auto" role="main">
|
||
|
||
<div class="topbar container-xl fixed-top">
|
||
<div class="topbar-contents row">
|
||
<div class="col-12 col-md-3 bd-topbar-whitespace site-navigation show"></div>
|
||
<div class="col pl-md-4 topbar-main">
|
||
|
||
<button id="navbar-toggler" class="navbar-toggler ml-0" type="button" data-toggle="collapse"
|
||
data-toggle="tooltip" data-placement="bottom" data-target=".site-navigation" aria-controls="navbar-menu"
|
||
aria-expanded="true" aria-label="Toggle navigation" aria-controls="site-navigation"
|
||
title="Toggle navigation" data-toggle="tooltip" data-placement="left">
|
||
<i class="fas fa-bars"></i>
|
||
<i class="fas fa-arrow-left"></i>
|
||
<i class="fas fa-arrow-up"></i>
|
||
</button>
|
||
|
||
|
||
<div class="dropdown-buttons-trigger">
|
||
<button id="dropdown-buttons-trigger" class="btn btn-secondary topbarbtn" aria-label="Download this page"><i
|
||
class="fas fa-download"></i></button>
|
||
|
||
<div class="dropdown-buttons">
|
||
<!-- ipynb file if we had a myst markdown file -->
|
||
|
||
<!-- Download raw file -->
|
||
<a class="dropdown-buttons" href="_sources/exercisesweek41.ipynb"><button type="button"
|
||
class="btn btn-secondary topbarbtn" title="Download source file" data-toggle="tooltip"
|
||
data-placement="left">.ipynb</button></a>
|
||
<!-- Download PDF via print -->
|
||
<button type="button" id="download-print" class="btn btn-secondary topbarbtn" title="Print to PDF"
|
||
onclick="printPdf(this)" data-toggle="tooltip" data-placement="left">.pdf</button>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Source interaction buttons -->
|
||
|
||
<!-- Full screen (wrap in <a> to have style consistency -->
|
||
|
||
<a class="full-screen-button"><button type="button" class="btn btn-secondary topbarbtn" data-toggle="tooltip"
|
||
data-placement="bottom" onclick="toggleFullScreen()" aria-label="Fullscreen mode"
|
||
title="Fullscreen mode"><i
|
||
class="fas fa-expand"></i></button></a>
|
||
|
||
<!-- Launch buttons -->
|
||
|
||
</div>
|
||
|
||
<!-- Table of contents -->
|
||
<div class="d-none d-md-block col-md-2 bd-toc show noprint">
|
||
|
||
<div class="tocsection onthispage pt-5 pb-3">
|
||
<i class="fas fa-list"></i> Contents
|
||
</div>
|
||
<nav id="bd-toc-nav" aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#">
|
||
Exercises week 41
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#overarching-aims-of-the-exercises-this-week">
|
||
Overarching aims of the exercises this week
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#code-examples-from-week-39-and-40">
|
||
Code examples from week 39 and 40
|
||
</a>
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#code-with-a-number-of-minibatches-which-varies-analytical-gradient">
|
||
Code with a Number of Minibatches which varies, analytical gradient
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#momentum-based-gd">
|
||
Momentum based GD
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#algorithms-and-codes-for-adagrad-rmsprop-and-adam">
|
||
Algorithms and codes for Adagrad, RMSprop and Adam
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#practical-tips">
|
||
Practical tips
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#using-automatic-differentation-with-ols">
|
||
Using Automatic differentation with OLS
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#same-code-but-now-with-momentum-gradient-descent">
|
||
Same code but now with momentum gradient descent
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#but-noen-of-these-can-compete-with-newton-s-method">
|
||
But noen of these can compete with Newton’s method
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#including-stochastic-gradient-descent-with-autograd">
|
||
Including Stochastic Gradient Descent with Autograd
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#id1">
|
||
Same code but now with momentum gradient descent
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#adagrad-algorithm-taken-from-goodfellow-et-al">
|
||
AdaGrad algorithm, taken from Goodfellow et al
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#similar-second-order-function-now-problem-but-now-with-adagrad">
|
||
Similar (second order function now) problem but now with AdaGrad
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#rmsprop-algorithm-taken-from-goodfellow-et-al">
|
||
RMSProp algorithm, taken from Goodfellow et al
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent">
|
||
RMSprop for adaptive learning rate with Stochastic Gradient Descent
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#adam-algorithm-taken-from-goodfellow-et-al">
|
||
ADAM algorithm, taken from Goodfellow et al
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#and-finally-adam">
|
||
And finally ADAM
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#introducing-jax">
|
||
Introducing JAX
|
||
</a>
|
||
<ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#getting-started-with-jax-note-the-way-we-import-numpy">
|
||
Getting started with Jax, note the way we import numpy
|
||
</a>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#a-warm-up-example">
|
||
A warm-up example
|
||
</a>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#a-more-advanced-example">
|
||
A more advanced example
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div id="main-content" class="row">
|
||
<div class="col-12 col-md-9 pl-md-3 pr-md-0">
|
||
<!-- Table of contents that is only displayed when printing the page -->
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>Exercises week 41</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#">
|
||
Exercises week 41
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#overarching-aims-of-the-exercises-this-week">
|
||
Overarching aims of the exercises this week
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#code-examples-from-week-39-and-40">
|
||
Code examples from week 39 and 40
|
||
</a>
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#code-with-a-number-of-minibatches-which-varies-analytical-gradient">
|
||
Code with a Number of Minibatches which varies, analytical gradient
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#momentum-based-gd">
|
||
Momentum based GD
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#algorithms-and-codes-for-adagrad-rmsprop-and-adam">
|
||
Algorithms and codes for Adagrad, RMSprop and Adam
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#practical-tips">
|
||
Practical tips
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#using-automatic-differentation-with-ols">
|
||
Using Automatic differentation with OLS
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#same-code-but-now-with-momentum-gradient-descent">
|
||
Same code but now with momentum gradient descent
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#but-noen-of-these-can-compete-with-newton-s-method">
|
||
But noen of these can compete with Newton’s method
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#including-stochastic-gradient-descent-with-autograd">
|
||
Including Stochastic Gradient Descent with Autograd
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#id1">
|
||
Same code but now with momentum gradient descent
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#adagrad-algorithm-taken-from-goodfellow-et-al">
|
||
AdaGrad algorithm, taken from Goodfellow et al
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#similar-second-order-function-now-problem-but-now-with-adagrad">
|
||
Similar (second order function now) problem but now with AdaGrad
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#rmsprop-algorithm-taken-from-goodfellow-et-al">
|
||
RMSProp algorithm, taken from Goodfellow et al
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent">
|
||
RMSprop for adaptive learning rate with Stochastic Gradient Descent
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#adam-algorithm-taken-from-goodfellow-et-al">
|
||
ADAM algorithm, taken from Goodfellow et al
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#and-finally-adam">
|
||
And finally ADAM
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#introducing-jax">
|
||
Introducing JAX
|
||
</a>
|
||
<ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#getting-started-with-jax-note-the-way-we-import-numpy">
|
||
Getting started with Jax, note the way we import numpy
|
||
</a>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#a-warm-up-example">
|
||
A warm-up example
|
||
</a>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#a-more-advanced-example">
|
||
A more advanced example
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div>
|
||
|
||
<!-- HTML file automatically generated from DocOnce source (https://github.com/doconce/doconce/)
|
||
doconce format html exercisesweek41.do.txt -->
|
||
<!-- dom:TITLE: Exercises week 41 --><div class="tex2jax_ignore mathjax_ignore section" id="exercises-week-41">
|
||
<h1>Exercises week 41<a class="headerlink" href="#exercises-week-41" title="Permalink to this headline">¶</a></h1>
|
||
<p><strong>October 4-11, 2024</strong></p>
|
||
<p>Date: <strong>Deadline is Friday October 11 at midnight</strong></p>
|
||
</div>
|
||
<div class="tex2jax_ignore mathjax_ignore section" id="overarching-aims-of-the-exercises-this-week">
|
||
<h1>Overarching aims of the exercises this week<a class="headerlink" href="#overarching-aims-of-the-exercises-this-week" title="Permalink to this headline">¶</a></h1>
|
||
<p>The aim of the exercises this week is to get started with implementing
|
||
gradient methods of relevance for project 2. This exercise will also
|
||
be continued next week with the addition of automatic differentation.
|
||
Everything you develop here will be used in project 2.</p>
|
||
<p>In order to get started, we will now replace in our standard ordinary
|
||
least squares (OLS) and Ridge regression codes (from project 1) the
|
||
matrix inversion algorithm with our own gradient descent (GD) and SGD
|
||
codes. You can use the Franke function or the terrain data from
|
||
project 1. <strong>However, we recommend using a simpler function like</strong>
|
||
<span class="math notranslate nohighlight">\(f(x)=a_0+a_1x+a_2x^2\)</span> or higher-order one-dimensional polynomials.
|
||
You can obviously test your final codes against for example the Franke
|
||
function. Automatic differentiation will be discussed next week.</p>
|
||
<p>You should include in your analysis of the GD and SGD codes the following elements</p>
|
||
<ol class="simple">
|
||
<li><p>A plain gradient descent with a fixed learning rate (you will need to tune it) using the analytical expression of the gradients</p></li>
|
||
<li><p>Add momentum to the plain GD code and compare convergence with a fixed learning rate (you may need to tune the learning rate), again using the analytical expression of the gradients.</p></li>
|
||
<li><p>Repeat these steps for stochastic gradient descent with mini batches and a given number of epochs. Use a tunable learning rate as discussed in the lectures from week 39. Discuss the results as functions of the various parameters (size of batches, number of epochs etc)</p></li>
|
||
<li><p>Implement the Adagrad method in order to tune the learning rate. Do this with and without momentum for plain gradient descent and SGD.</p></li>
|
||
<li><p>Add RMSprop and Adam to your library of methods for tuning the learning rate.</p></li>
|
||
</ol>
|
||
<p>The lecture notes from weeks 39 and 40 contain more information and code examples. Feel free to use these examples.</p>
|
||
<p>In summary, you should
|
||
perform an analysis of the results for OLS and Ridge regression as
|
||
function of the chosen learning rates, the number of mini-batches and
|
||
epochs as well as algorithm for scaling the learning rate. You can
|
||
also compare your own results with those that can be obtained using
|
||
for example <strong>Scikit-Learn</strong>’s various SGD options. Discuss your
|
||
results. For Ridge regression you need now to study the results as functions of the hyper-parameter <span class="math notranslate nohighlight">\(\lambda\)</span> and
|
||
the learning rate <span class="math notranslate nohighlight">\(\eta\)</span>. Discuss your results.</p>
|
||
<p>You will need your SGD code for the setup of the Neural Network and
|
||
Logistic Regression codes. You will find the Python <a class="reference external" href="https://seaborn.pydata.org/generated/seaborn.heatmap.html">Seaborn
|
||
package</a>
|
||
useful when plotting the results as function of the learning rate
|
||
<span class="math notranslate nohighlight">\(\eta\)</span> and the hyper-parameter <span class="math notranslate nohighlight">\(\lambda\)</span> when you use Ridge
|
||
regression.</p>
|
||
<p>We recommend reading chapter 8 on optimization from the textbook of <a class="reference external" href="https://www.deeplearningbook.org/">Goodfellow, Bengio and Courville</a>. This chapter contains many useful insights and discussions on the optimization part of machine learning.</p>
|
||
</div>
|
||
<div class="tex2jax_ignore mathjax_ignore section" id="code-examples-from-week-39-and-40">
|
||
<h1>Code examples from week 39 and 40<a class="headerlink" href="#code-examples-from-week-39-and-40" title="Permalink to this headline">¶</a></h1>
|
||
<div class="section" id="code-with-a-number-of-minibatches-which-varies-analytical-gradient">
|
||
<h2>Code with a Number of Minibatches which varies, analytical gradient<a class="headerlink" href="#code-with-a-number-of-minibatches-which-varies-analytical-gradient" title="Permalink to this headline">¶</a></h2>
|
||
<p>In the code here we vary the number of mini-batches.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="o">%</span><span class="k">matplotlib</span> inline
|
||
|
||
<span class="c1"># Importing various packages</span>
|
||
<span class="kn">from</span> <span class="nn">math</span> <span class="kn">import</span> <span class="n">exp</span><span class="p">,</span> <span class="n">sqrt</span>
|
||
<span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span><span class="p">,</span> <span class="n">seed</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="mi">2</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mi">4</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">c_</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)),</span> <span class="n">x</span><span class="p">]</span>
|
||
<span class="n">XT_X</span> <span class="o">=</span> <span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X</span>
|
||
<span class="n">theta_linreg</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">inv</span><span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X</span><span class="p">)</span> <span class="o">@</span> <span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">y</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Own inversion"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta_linreg</span><span class="p">)</span>
|
||
<span class="c1"># Hessian matrix</span>
|
||
<span class="n">H</span> <span class="o">=</span> <span class="p">(</span><span class="mf">2.0</span><span class="o">/</span><span class="n">n</span><span class="p">)</span><span class="o">*</span> <span class="n">XT_X</span>
|
||
<span class="n">EigValues</span><span class="p">,</span> <span class="n">EigVectors</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">eig</span><span class="p">(</span><span class="n">H</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Eigenvalues of Hessian Matrix:</span><span class="si">{</span><span class="n">EigValues</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
|
||
<span class="n">theta</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">eta</span> <span class="o">=</span> <span class="mf">1.0</span><span class="o">/</span><span class="n">np</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">EigValues</span><span class="p">)</span>
|
||
<span class="n">Niterations</span> <span class="o">=</span> <span class="mi">1000</span>
|
||
|
||
<span class="c1">#while (iter <= Ni... or test)</span>
|
||
<span class="k">for</span> <span class="nb">iter</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">Niterations</span><span class="p">):</span>
|
||
<span class="n">gradients</span> <span class="o">=</span> <span class="mf">2.0</span><span class="o">/</span><span class="n">n</span><span class="o">*</span><span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="p">((</span><span class="n">X</span> <span class="o">@</span> <span class="n">theta</span><span class="p">)</span><span class="o">-</span><span class="n">y</span><span class="p">)</span>
|
||
<span class="n">theta</span> <span class="o">-=</span> <span class="n">eta</span><span class="o">*</span><span class="n">gradients</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"theta from own gd"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
|
||
<span class="n">xnew</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">0</span><span class="p">],[</span><span class="mi">2</span><span class="p">]])</span>
|
||
<span class="n">Xnew</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">c_</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span><span class="mi">1</span><span class="p">)),</span> <span class="n">xnew</span><span class="p">]</span>
|
||
<span class="n">ypredict</span> <span class="o">=</span> <span class="n">Xnew</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
<span class="n">ypredict2</span> <span class="o">=</span> <span class="n">Xnew</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">theta_linreg</span><span class="p">)</span>
|
||
|
||
<span class="n">n_epochs</span> <span class="o">=</span> <span class="mi">50</span>
|
||
<span class="n">M</span> <span class="o">=</span> <span class="mi">5</span> <span class="c1">#size of each minibatch</span>
|
||
<span class="n">m</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">n</span><span class="o">/</span><span class="n">M</span><span class="p">)</span> <span class="c1">#number of minibatches</span>
|
||
<span class="n">t0</span><span class="p">,</span> <span class="n">t1</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">50</span>
|
||
|
||
<span class="k">def</span> <span class="nf">learning_schedule</span><span class="p">(</span><span class="n">t</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">t0</span><span class="o">/</span><span class="p">(</span><span class="n">t</span><span class="o">+</span><span class="n">t1</span><span class="p">)</span>
|
||
|
||
<span class="n">theta</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_epochs</span><span class="p">):</span>
|
||
<span class="c1"># Can you figure out a better way of setting up the contributions to each batch?</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">m</span><span class="p">):</span>
|
||
<span class="n">random_index</span> <span class="o">=</span> <span class="n">M</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="n">m</span><span class="p">)</span>
|
||
<span class="n">xi</span> <span class="o">=</span> <span class="n">X</span><span class="p">[</span><span class="n">random_index</span><span class="p">:</span><span class="n">random_index</span><span class="o">+</span><span class="n">M</span><span class="p">]</span>
|
||
<span class="n">yi</span> <span class="o">=</span> <span class="n">y</span><span class="p">[</span><span class="n">random_index</span><span class="p">:</span><span class="n">random_index</span><span class="o">+</span><span class="n">M</span><span class="p">]</span>
|
||
<span class="n">gradients</span> <span class="o">=</span> <span class="p">(</span><span class="mf">2.0</span><span class="o">/</span><span class="n">M</span><span class="p">)</span><span class="o">*</span> <span class="n">xi</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="p">((</span><span class="n">xi</span> <span class="o">@</span> <span class="n">theta</span><span class="p">)</span><span class="o">-</span><span class="n">yi</span><span class="p">)</span>
|
||
<span class="n">eta</span> <span class="o">=</span> <span class="n">learning_schedule</span><span class="p">(</span><span class="n">epoch</span><span class="o">*</span><span class="n">m</span><span class="o">+</span><span class="n">i</span><span class="p">)</span>
|
||
<span class="n">theta</span> <span class="o">=</span> <span class="n">theta</span> <span class="o">-</span> <span class="n">eta</span><span class="o">*</span><span class="n">gradients</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"theta from own sdg"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">xnew</span><span class="p">,</span> <span class="n">ypredict</span><span class="p">,</span> <span class="s2">"r-"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">xnew</span><span class="p">,</span> <span class="n">ypredict2</span><span class="p">,</span> <span class="s2">"b-"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span> <span class="p">,</span><span class="s1">'ro'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">axis</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span><span class="mf">2.0</span><span class="p">,</span><span class="mi">0</span><span class="p">,</span> <span class="mf">15.0</span><span class="p">])</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="sa">r</span><span class="s1">'$x$'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="sa">r</span><span class="s1">'$y$'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="sa">r</span><span class="s1">'Random numbers '</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
|
||
[[4.15515965]
|
||
[2.99763835]]
|
||
Eigenvalues of Hessian Matrix:[0.3156062 4.19886862]
|
||
theta from own gd
|
||
[[4.15515965]
|
||
[2.99763835]]
|
||
theta from own sdg
|
||
[[4.18290649]
|
||
[2.96486591]]
|
||
</pre></div>
|
||
</div>
|
||
<img alt="_images/exercisesweek41_5_1.png" src="_images/exercisesweek41_5_1.png" />
|
||
</div>
|
||
</div>
|
||
<p>In the above code, we have use replacement in setting up the
|
||
mini-batches. The discussion
|
||
<a class="reference external" href="https://sebastianraschka.com/faq/docs/sgd-methods.html">here</a> may be
|
||
useful.</p>
|
||
</div>
|
||
<div class="section" id="momentum-based-gd">
|
||
<h2>Momentum based GD<a class="headerlink" href="#momentum-based-gd" title="Permalink to this headline">¶</a></h2>
|
||
<p>The stochastic gradient descent (SGD) is almost always used with a
|
||
<em>momentum</em> or inertia term that serves as a memory of the direction we
|
||
are moving in parameter space. This is typically implemented as
|
||
follows</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathbf{v}_{t}=\gamma \mathbf{v}_{t-1}+\eta_{t}\nabla_\theta E(\boldsymbol{\theta}_t) \nonumber
|
||
\]</div>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto1"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
\boldsymbol{\theta}_{t+1}= \boldsymbol{\theta}_t -\mathbf{v}_{t},
|
||
\label{_auto1} \tag{1}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>where we have introduced a momentum parameter <span class="math notranslate nohighlight">\(\gamma\)</span>, with
|
||
<span class="math notranslate nohighlight">\(0\le\gamma\le 1\)</span>, and for brevity we dropped the explicit notation to
|
||
indicate the gradient is to be taken over a different mini-batch at
|
||
each step. We call this algorithm gradient descent with momentum
|
||
(GDM). From these equations, it is clear that <span class="math notranslate nohighlight">\(\mathbf{v}_t\)</span> is a
|
||
running average of recently encountered gradients and
|
||
<span class="math notranslate nohighlight">\((1-\gamma)^{-1}\)</span> sets the characteristic time scale for the memory
|
||
used in the averaging procedure. Consistent with this, when
|
||
<span class="math notranslate nohighlight">\(\gamma=0\)</span>, this just reduces down to ordinary SGD as discussed
|
||
earlier. An equivalent way of writing the updates is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\Delta \boldsymbol{\theta}_{t+1} = \gamma \Delta \boldsymbol{\theta}_t -\ \eta_{t}\nabla_\theta E(\boldsymbol{\theta}_t),
|
||
\]</div>
|
||
<p>where we have defined <span class="math notranslate nohighlight">\(\Delta \boldsymbol{\theta}_{t}= \boldsymbol{\theta}_t-\boldsymbol{\theta}_{t-1}\)</span>.</p>
|
||
</div>
|
||
<div class="section" id="algorithms-and-codes-for-adagrad-rmsprop-and-adam">
|
||
<h2>Algorithms and codes for Adagrad, RMSprop and Adam<a class="headerlink" href="#algorithms-and-codes-for-adagrad-rmsprop-and-adam" title="Permalink to this headline">¶</a></h2>
|
||
<p>The algorithms we have implemented are well described in the text by <a class="reference external" href="https://www.deeplearningbook.org/contents/optimization.html">Goodfellow, Bengio and Courville, chapter 8</a>.</p>
|
||
<p>The codes which implement these algorithms are discussed after our presentation of automatic differentiation.</p>
|
||
</div>
|
||
<div class="section" id="practical-tips">
|
||
<h2>Practical tips<a class="headerlink" href="#practical-tips" title="Permalink to this headline">¶</a></h2>
|
||
<ul class="simple">
|
||
<li><p><strong>Randomize the data when making mini-batches</strong>. It is always important to randomly shuffle the data when forming mini-batches. Otherwise, the gradient descent method can fit spurious correlations resulting from the order in which data is presented.</p></li>
|
||
<li><p><strong>Transform your inputs</strong>. Learning becomes difficult when our landscape has a mixture of steep and flat directions. One simple trick for minimizing these situations is to standardize the data by subtracting the mean and normalizing the variance of input variables. Whenever possible, also decorrelate the inputs. To understand why this is helpful, consider the case of linear regression. It is easy to show that for the squared error cost function, the Hessian of the cost function is just the correlation matrix between the inputs. Thus, by standardizing the inputs, we are ensuring that the landscape looks homogeneous in all directions in parameter space. Since most deep networks can be viewed as linear transformations followed by a non-linearity at each layer, we expect this intuition to hold beyond the linear case.</p></li>
|
||
<li><p><strong>Monitor the out-of-sample performance.</strong> Always monitor the performance of your model on a validation set (a small portion of the training data that is held out of the training process to serve as a proxy for the test set. If the validation error starts increasing, then the model is beginning to overfit. Terminate the learning process. This <em>early stopping</em> significantly improves performance in many settings.</p></li>
|
||
<li><p><strong>Adaptive optimization methods don’t always have good generalization.</strong> Recent studies have shown that adaptive methods such as ADAM, RMSPorp, and AdaGrad tend to have poor generalization compared to SGD or SGD with momentum, particularly in the high-dimensional limit (i.e. the number of parameters exceeds the number of data points). Although it is not clear at this stage why these methods perform so well in training deep neural networks, simpler procedures like properly-tuned SGD may work as well or better in these applications.</p></li>
|
||
</ul>
|
||
<p>Geron’s text, see chapter 11, has several interesting discussions.</p>
|
||
</div>
|
||
<div class="section" id="using-automatic-differentation-with-ols">
|
||
<h2>Using Automatic differentation with OLS<a class="headerlink" href="#using-automatic-differentation-with-ols" title="Permalink to this headline">¶</a></h2>
|
||
<p>We conclude the part on optmization by showing how we can make codes
|
||
for linear regression and logistic regression using <strong>autograd</strong>. The
|
||
first example shows results with ordinary leats squares.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Using Autograd to calculate gradients for OLS</span>
|
||
<span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span><span class="p">,</span> <span class="n">seed</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">autograd.numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">autograd</span> <span class="kn">import</span> <span class="n">grad</span>
|
||
|
||
<span class="k">def</span> <span class="nf">CostOLS</span><span class="p">(</span><span class="n">beta</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="p">(</span><span class="mf">1.0</span><span class="o">/</span><span class="n">n</span><span class="p">)</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y</span><span class="o">-</span><span class="n">X</span> <span class="o">@</span> <span class="n">beta</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span>
|
||
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="mi">2</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mi">4</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">c_</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)),</span> <span class="n">x</span><span class="p">]</span>
|
||
<span class="n">XT_X</span> <span class="o">=</span> <span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X</span>
|
||
<span class="n">theta_linreg</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">pinv</span><span class="p">(</span><span class="n">XT_X</span><span class="p">)</span> <span class="o">@</span> <span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">y</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Own inversion"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta_linreg</span><span class="p">)</span>
|
||
<span class="c1"># Hessian matrix</span>
|
||
<span class="n">H</span> <span class="o">=</span> <span class="p">(</span><span class="mf">2.0</span><span class="o">/</span><span class="n">n</span><span class="p">)</span><span class="o">*</span> <span class="n">XT_X</span>
|
||
<span class="n">EigValues</span><span class="p">,</span> <span class="n">EigVectors</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">eig</span><span class="p">(</span><span class="n">H</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Eigenvalues of Hessian Matrix:</span><span class="si">{</span><span class="n">EigValues</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
|
||
<span class="n">theta</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">eta</span> <span class="o">=</span> <span class="mf">1.0</span><span class="o">/</span><span class="n">np</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">EigValues</span><span class="p">)</span>
|
||
<span class="n">Niterations</span> <span class="o">=</span> <span class="mi">1000</span>
|
||
<span class="c1"># define the gradient</span>
|
||
<span class="n">training_gradient</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">CostOLS</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="nb">iter</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">Niterations</span><span class="p">):</span>
|
||
<span class="n">gradients</span> <span class="o">=</span> <span class="n">training_gradient</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
<span class="n">theta</span> <span class="o">-=</span> <span class="n">eta</span><span class="o">*</span><span class="n">gradients</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"theta from own gd"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
|
||
<span class="n">xnew</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">0</span><span class="p">],[</span><span class="mi">2</span><span class="p">]])</span>
|
||
<span class="n">Xnew</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">c_</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span><span class="mi">1</span><span class="p">)),</span> <span class="n">xnew</span><span class="p">]</span>
|
||
<span class="n">ypredict</span> <span class="o">=</span> <span class="n">Xnew</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
<span class="n">ypredict2</span> <span class="o">=</span> <span class="n">Xnew</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">theta_linreg</span><span class="p">)</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">xnew</span><span class="p">,</span> <span class="n">ypredict</span><span class="p">,</span> <span class="s2">"r-"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">xnew</span><span class="p">,</span> <span class="n">ypredict2</span><span class="p">,</span> <span class="s2">"b-"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span> <span class="p">,</span><span class="s1">'ro'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">axis</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span><span class="mf">2.0</span><span class="p">,</span><span class="mi">0</span><span class="p">,</span> <span class="mf">15.0</span><span class="p">])</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="sa">r</span><span class="s1">'$x$'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="sa">r</span><span class="s1">'$y$'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="sa">r</span><span class="s1">'Random numbers '</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
|
||
[[3.98635543]
|
||
[2.9642492 ]]
|
||
Eigenvalues of Hessian Matrix:[0.31545424 4.4234122 ]
|
||
theta from own gd
|
||
[[3.98635543]
|
||
[2.9642492 ]]
|
||
</pre></div>
|
||
</div>
|
||
<img alt="_images/exercisesweek41_16_1.png" src="_images/exercisesweek41_16_1.png" />
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="same-code-but-now-with-momentum-gradient-descent">
|
||
<h2>Same code but now with momentum gradient descent<a class="headerlink" href="#same-code-but-now-with-momentum-gradient-descent" title="Permalink to this headline">¶</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Using Autograd to calculate gradients for OLS</span>
|
||
<span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span><span class="p">,</span> <span class="n">seed</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">autograd.numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">autograd</span> <span class="kn">import</span> <span class="n">grad</span>
|
||
|
||
<span class="k">def</span> <span class="nf">CostOLS</span><span class="p">(</span><span class="n">beta</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="p">(</span><span class="mf">1.0</span><span class="o">/</span><span class="n">n</span><span class="p">)</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y</span><span class="o">-</span><span class="n">X</span> <span class="o">@</span> <span class="n">beta</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span>
|
||
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="mi">2</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mi">4</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span><span class="c1">#+np.random.randn(n,1)</span>
|
||
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">c_</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)),</span> <span class="n">x</span><span class="p">]</span>
|
||
<span class="n">XT_X</span> <span class="o">=</span> <span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X</span>
|
||
<span class="n">theta_linreg</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">pinv</span><span class="p">(</span><span class="n">XT_X</span><span class="p">)</span> <span class="o">@</span> <span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">y</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Own inversion"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta_linreg</span><span class="p">)</span>
|
||
<span class="c1"># Hessian matrix</span>
|
||
<span class="n">H</span> <span class="o">=</span> <span class="p">(</span><span class="mf">2.0</span><span class="o">/</span><span class="n">n</span><span class="p">)</span><span class="o">*</span> <span class="n">XT_X</span>
|
||
<span class="n">EigValues</span><span class="p">,</span> <span class="n">EigVectors</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">eig</span><span class="p">(</span><span class="n">H</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Eigenvalues of Hessian Matrix:</span><span class="si">{</span><span class="n">EigValues</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
|
||
<span class="n">theta</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">eta</span> <span class="o">=</span> <span class="mf">1.0</span><span class="o">/</span><span class="n">np</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">EigValues</span><span class="p">)</span>
|
||
<span class="n">Niterations</span> <span class="o">=</span> <span class="mi">30</span>
|
||
|
||
<span class="c1"># define the gradient</span>
|
||
<span class="n">training_gradient</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">CostOLS</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="nb">iter</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">Niterations</span><span class="p">):</span>
|
||
<span class="n">gradients</span> <span class="o">=</span> <span class="n">training_gradient</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
<span class="n">theta</span> <span class="o">-=</span> <span class="n">eta</span><span class="o">*</span><span class="n">gradients</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="nb">iter</span><span class="p">,</span><span class="n">gradients</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span><span class="n">gradients</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"theta from own gd"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Now improve with momentum gradient descent</span>
|
||
<span class="n">change</span> <span class="o">=</span> <span class="mf">0.0</span>
|
||
<span class="n">delta_momentum</span> <span class="o">=</span> <span class="mf">0.3</span>
|
||
<span class="k">for</span> <span class="nb">iter</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">Niterations</span><span class="p">):</span>
|
||
<span class="c1"># calculate gradient</span>
|
||
<span class="n">gradients</span> <span class="o">=</span> <span class="n">training_gradient</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
<span class="c1"># calculate update</span>
|
||
<span class="n">new_change</span> <span class="o">=</span> <span class="n">eta</span><span class="o">*</span><span class="n">gradients</span><span class="o">+</span><span class="n">delta_momentum</span><span class="o">*</span><span class="n">change</span>
|
||
<span class="c1"># take a step</span>
|
||
<span class="n">theta</span> <span class="o">-=</span> <span class="n">new_change</span>
|
||
<span class="c1"># save the change</span>
|
||
<span class="n">change</span> <span class="o">=</span> <span class="n">new_change</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="nb">iter</span><span class="p">,</span><span class="n">gradients</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span><span class="n">gradients</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"theta from own gd wth momentum"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
|
||
[[4.]
|
||
[3.]]
|
||
Eigenvalues of Hessian Matrix:[0.35350927 4.2330123 ]
|
||
0 [-10.02028415] [-11.38281882]
|
||
1 [-0.12978318] [0.11144292]
|
||
2 [-0.11894467] [0.10213605]
|
||
3 [-0.10901131] [0.09360642]
|
||
4 [-0.0999075] [0.08578912]
|
||
5 [-0.09156398] [0.07862466]
|
||
6 [-0.08391725] [0.07205852]
|
||
7 [-0.07690911] [0.06604073]
|
||
8 [-0.07048624] [0.06052551]
|
||
9 [-0.06459976] [0.05547088]
|
||
10 [-0.05920488] [0.05083837]
|
||
11 [-0.05426053] [0.04659273]
|
||
12 [-0.0497291] [0.04270166]
|
||
13 [-0.0455761] [0.03913554]
|
||
14 [-0.04176993] [0.03586723]
|
||
15 [-0.03828162] [0.03287187]
|
||
16 [-0.03508463] [0.03012666]
|
||
17 [-0.03215463] [0.02761071]
|
||
18 [-0.02946931] [0.02530487]
|
||
19 [-0.02700826] [0.0231916]
|
||
20 [-0.02475273] [0.02125481]
|
||
21 [-0.02268557] [0.01947977]
|
||
22 [-0.02079104] [0.01785297]
|
||
23 [-0.01905473] [0.01636202]
|
||
24 [-0.01746342] [0.01499559]
|
||
25 [-0.01600501] [0.01374327]
|
||
26 [-0.01466839] [0.01259554]
|
||
27 [-0.0134434] [0.01154365]
|
||
28 [-0.01232071] [0.01057961]
|
||
29 [-0.01129178] [0.00969608]
|
||
theta from own gd
|
||
[[3.9707256]
|
||
[3.0251375]]
|
||
0 [-0.01034877] [0.00888634]
|
||
1 [-0.00948452] [0.00814422]
|
||
2 [-0.00843317] [0.00724144]
|
||
3 [-0.00741349] [0.00636586]
|
||
4 [-0.00648847] [0.00557155]
|
||
5 [-0.00566909] [0.00486797]
|
||
6 [-0.00494984] [0.00425036]
|
||
7 [-0.00432069] [0.00371011]
|
||
8 [-0.00377111] [0.0032382]
|
||
9 [-0.00329131] [0.0028262]
|
||
10 [-0.0028725] [0.00246657]
|
||
11 [-0.00250697] [0.0021527]
|
||
12 [-0.00218794] [0.00187876]
|
||
13 [-0.00190952] [0.00163967]
|
||
14 [-0.00166652] [0.00143102]
|
||
15 [-0.00145445] [0.00124891]
|
||
16 [-0.00126936] [0.00108998]
|
||
17 [-0.00110783] [0.00095127]
|
||
18 [-0.00096685] [0.00083022]
|
||
19 [-0.00084381] [0.00072457]
|
||
20 [-0.00073643] [0.00063236]
|
||
21 [-0.00064272] [0.00055189]
|
||
22 [-0.00056093] [0.00048166]
|
||
23 [-0.00048955] [0.00042037]
|
||
24 [-0.00042725] [0.00036687]
|
||
25 [-0.00037288] [0.00032019]
|
||
26 [-0.00032543] [0.00027944]
|
||
27 [-0.00028402] [0.00024388]
|
||
28 [-0.00024787] [0.00021284]
|
||
29 [-0.00021633] [0.00018576]
|
||
theta from own gd wth momentum
|
||
[[3.99946593]
|
||
[3.0004586 ]]
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="but-noen-of-these-can-compete-with-newton-s-method">
|
||
<h2>But noen of these can compete with Newton’s method<a class="headerlink" href="#but-noen-of-these-can-compete-with-newton-s-method" title="Permalink to this headline">¶</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Using Newton's method</span>
|
||
<span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span><span class="p">,</span> <span class="n">seed</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">autograd.numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">autograd</span> <span class="kn">import</span> <span class="n">grad</span>
|
||
|
||
<span class="k">def</span> <span class="nf">CostOLS</span><span class="p">(</span><span class="n">beta</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="p">(</span><span class="mf">1.0</span><span class="o">/</span><span class="n">n</span><span class="p">)</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y</span><span class="o">-</span><span class="n">X</span> <span class="o">@</span> <span class="n">beta</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span>
|
||
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="mi">2</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mi">4</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">c_</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)),</span> <span class="n">x</span><span class="p">]</span>
|
||
<span class="n">XT_X</span> <span class="o">=</span> <span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X</span>
|
||
<span class="n">beta_linreg</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">pinv</span><span class="p">(</span><span class="n">XT_X</span><span class="p">)</span> <span class="o">@</span> <span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">y</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Own inversion"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">beta_linreg</span><span class="p">)</span>
|
||
<span class="c1"># Hessian matrix</span>
|
||
<span class="n">H</span> <span class="o">=</span> <span class="p">(</span><span class="mf">2.0</span><span class="o">/</span><span class="n">n</span><span class="p">)</span><span class="o">*</span> <span class="n">XT_X</span>
|
||
<span class="c1"># Note that here the Hessian does not depend on the parameters beta</span>
|
||
<span class="n">invH</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">pinv</span><span class="p">(</span><span class="n">H</span><span class="p">)</span>
|
||
<span class="n">EigValues</span><span class="p">,</span> <span class="n">EigVectors</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">eig</span><span class="p">(</span><span class="n">H</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Eigenvalues of Hessian Matrix:</span><span class="si">{</span><span class="n">EigValues</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
|
||
<span class="n">beta</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">Niterations</span> <span class="o">=</span> <span class="mi">5</span>
|
||
|
||
<span class="c1"># define the gradient</span>
|
||
<span class="n">training_gradient</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">CostOLS</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="nb">iter</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">Niterations</span><span class="p">):</span>
|
||
<span class="n">gradients</span> <span class="o">=</span> <span class="n">training_gradient</span><span class="p">(</span><span class="n">beta</span><span class="p">)</span>
|
||
<span class="n">beta</span> <span class="o">-=</span> <span class="n">invH</span> <span class="o">@</span> <span class="n">gradients</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="nb">iter</span><span class="p">,</span><span class="n">gradients</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span><span class="n">gradients</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"beta from own Newton code"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">beta</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
|
||
[[4.16849001]
|
||
[3.03642937]]
|
||
Eigenvalues of Hessian Matrix:[0.3039241 4.51493779]
|
||
0 [-12.93402104] [-15.39497785]
|
||
1 [9.11944131e-15] [7.63687445e-15]
|
||
2 [2.11636264e-16] [3.19670115e-16]
|
||
3 [2.11636264e-16] [3.19670115e-16]
|
||
4 [2.11636264e-16] [3.19670115e-16]
|
||
beta from own Newton code
|
||
[[4.16849001]
|
||
[3.03642937]]
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="including-stochastic-gradient-descent-with-autograd">
|
||
<h2>Including Stochastic Gradient Descent with Autograd<a class="headerlink" href="#including-stochastic-gradient-descent-with-autograd" title="Permalink to this headline">¶</a></h2>
|
||
<p>In this code we include the stochastic gradient descent approach discussed above. Note here that we specify which argument we are taking the derivative with respect to when using <strong>autograd</strong>.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Using Autograd to calculate gradients using SGD</span>
|
||
<span class="c1"># OLS example</span>
|
||
<span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span><span class="p">,</span> <span class="n">seed</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">autograd.numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">autograd</span> <span class="kn">import</span> <span class="n">grad</span>
|
||
|
||
<span class="c1"># Note change from previous example</span>
|
||
<span class="k">def</span> <span class="nf">CostOLS</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">X</span><span class="p">,</span><span class="n">theta</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y</span><span class="o">-</span><span class="n">X</span> <span class="o">@</span> <span class="n">theta</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span>
|
||
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="mi">2</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mi">4</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">c_</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)),</span> <span class="n">x</span><span class="p">]</span>
|
||
<span class="n">XT_X</span> <span class="o">=</span> <span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X</span>
|
||
<span class="n">theta_linreg</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">pinv</span><span class="p">(</span><span class="n">XT_X</span><span class="p">)</span> <span class="o">@</span> <span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">y</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Own inversion"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta_linreg</span><span class="p">)</span>
|
||
<span class="c1"># Hessian matrix</span>
|
||
<span class="n">H</span> <span class="o">=</span> <span class="p">(</span><span class="mf">2.0</span><span class="o">/</span><span class="n">n</span><span class="p">)</span><span class="o">*</span> <span class="n">XT_X</span>
|
||
<span class="n">EigValues</span><span class="p">,</span> <span class="n">EigVectors</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">eig</span><span class="p">(</span><span class="n">H</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Eigenvalues of Hessian Matrix:</span><span class="si">{</span><span class="n">EigValues</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
|
||
<span class="n">theta</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">eta</span> <span class="o">=</span> <span class="mf">1.0</span><span class="o">/</span><span class="n">np</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">EigValues</span><span class="p">)</span>
|
||
<span class="n">Niterations</span> <span class="o">=</span> <span class="mi">1000</span>
|
||
|
||
<span class="c1"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
|
||
<span class="n">training_gradient</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">CostOLS</span><span class="p">,</span><span class="mi">2</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="nb">iter</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">Niterations</span><span class="p">):</span>
|
||
<span class="n">gradients</span> <span class="o">=</span> <span class="p">(</span><span class="mf">1.0</span><span class="o">/</span><span class="n">n</span><span class="p">)</span><span class="o">*</span><span class="n">training_gradient</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">X</span><span class="p">,</span> <span class="n">theta</span><span class="p">)</span>
|
||
<span class="n">theta</span> <span class="o">-=</span> <span class="n">eta</span><span class="o">*</span><span class="n">gradients</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"theta from own gd"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
|
||
<span class="n">xnew</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">0</span><span class="p">],[</span><span class="mi">2</span><span class="p">]])</span>
|
||
<span class="n">Xnew</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">c_</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span><span class="mi">1</span><span class="p">)),</span> <span class="n">xnew</span><span class="p">]</span>
|
||
<span class="n">ypredict</span> <span class="o">=</span> <span class="n">Xnew</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
<span class="n">ypredict2</span> <span class="o">=</span> <span class="n">Xnew</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">theta_linreg</span><span class="p">)</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">xnew</span><span class="p">,</span> <span class="n">ypredict</span><span class="p">,</span> <span class="s2">"r-"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">xnew</span><span class="p">,</span> <span class="n">ypredict2</span><span class="p">,</span> <span class="s2">"b-"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span> <span class="p">,</span><span class="s1">'ro'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">axis</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span><span class="mf">2.0</span><span class="p">,</span><span class="mi">0</span><span class="p">,</span> <span class="mf">15.0</span><span class="p">])</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="sa">r</span><span class="s1">'$x$'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="sa">r</span><span class="s1">'$y$'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="sa">r</span><span class="s1">'Random numbers '</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
|
||
<span class="n">n_epochs</span> <span class="o">=</span> <span class="mi">50</span>
|
||
<span class="n">M</span> <span class="o">=</span> <span class="mi">5</span> <span class="c1">#size of each minibatch</span>
|
||
<span class="n">m</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">n</span><span class="o">/</span><span class="n">M</span><span class="p">)</span> <span class="c1">#number of minibatches</span>
|
||
<span class="n">t0</span><span class="p">,</span> <span class="n">t1</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">50</span>
|
||
<span class="k">def</span> <span class="nf">learning_schedule</span><span class="p">(</span><span class="n">t</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">t0</span><span class="o">/</span><span class="p">(</span><span class="n">t</span><span class="o">+</span><span class="n">t1</span><span class="p">)</span>
|
||
|
||
<span class="n">theta</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_epochs</span><span class="p">):</span>
|
||
<span class="c1"># Can you figure out a better way of setting up the contributions to each batch?</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">m</span><span class="p">):</span>
|
||
<span class="n">random_index</span> <span class="o">=</span> <span class="n">M</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="n">m</span><span class="p">)</span>
|
||
<span class="n">xi</span> <span class="o">=</span> <span class="n">X</span><span class="p">[</span><span class="n">random_index</span><span class="p">:</span><span class="n">random_index</span><span class="o">+</span><span class="n">M</span><span class="p">]</span>
|
||
<span class="n">yi</span> <span class="o">=</span> <span class="n">y</span><span class="p">[</span><span class="n">random_index</span><span class="p">:</span><span class="n">random_index</span><span class="o">+</span><span class="n">M</span><span class="p">]</span>
|
||
<span class="n">gradients</span> <span class="o">=</span> <span class="p">(</span><span class="mf">1.0</span><span class="o">/</span><span class="n">M</span><span class="p">)</span><span class="o">*</span><span class="n">training_gradient</span><span class="p">(</span><span class="n">yi</span><span class="p">,</span> <span class="n">xi</span><span class="p">,</span> <span class="n">theta</span><span class="p">)</span>
|
||
<span class="n">eta</span> <span class="o">=</span> <span class="n">learning_schedule</span><span class="p">(</span><span class="n">epoch</span><span class="o">*</span><span class="n">m</span><span class="o">+</span><span class="n">i</span><span class="p">)</span>
|
||
<span class="n">theta</span> <span class="o">=</span> <span class="n">theta</span> <span class="o">-</span> <span class="n">eta</span><span class="o">*</span><span class="n">gradients</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"theta from own sdg"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
|
||
[[4.37133465]
|
||
[2.74809387]]
|
||
Eigenvalues of Hessian Matrix:[0.32760411 4.2384333 ]
|
||
</pre></div>
|
||
</div>
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>theta from own gd
|
||
[[4.37133465]
|
||
[2.74809387]]
|
||
</pre></div>
|
||
</div>
|
||
<img alt="_images/exercisesweek41_22_2.png" src="_images/exercisesweek41_22_2.png" />
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>theta from own sdg
|
||
[[4.31985268]
|
||
[2.72067593]]
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="id1">
|
||
<h2>Same code but now with momentum gradient descent<a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Using Autograd to calculate gradients using SGD</span>
|
||
<span class="c1"># OLS example</span>
|
||
<span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span><span class="p">,</span> <span class="n">seed</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">autograd.numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">autograd</span> <span class="kn">import</span> <span class="n">grad</span>
|
||
|
||
<span class="c1"># Note change from previous example</span>
|
||
<span class="k">def</span> <span class="nf">CostOLS</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">X</span><span class="p">,</span><span class="n">theta</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y</span><span class="o">-</span><span class="n">X</span> <span class="o">@</span> <span class="n">theta</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span>
|
||
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="mi">2</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mi">4</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">c_</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)),</span> <span class="n">x</span><span class="p">]</span>
|
||
<span class="n">XT_X</span> <span class="o">=</span> <span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X</span>
|
||
<span class="n">theta_linreg</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">pinv</span><span class="p">(</span><span class="n">XT_X</span><span class="p">)</span> <span class="o">@</span> <span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">y</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Own inversion"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta_linreg</span><span class="p">)</span>
|
||
<span class="c1"># Hessian matrix</span>
|
||
<span class="n">H</span> <span class="o">=</span> <span class="p">(</span><span class="mf">2.0</span><span class="o">/</span><span class="n">n</span><span class="p">)</span><span class="o">*</span> <span class="n">XT_X</span>
|
||
<span class="n">EigValues</span><span class="p">,</span> <span class="n">EigVectors</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">eig</span><span class="p">(</span><span class="n">H</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Eigenvalues of Hessian Matrix:</span><span class="si">{</span><span class="n">EigValues</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
|
||
<span class="n">theta</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">eta</span> <span class="o">=</span> <span class="mf">1.0</span><span class="o">/</span><span class="n">np</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">EigValues</span><span class="p">)</span>
|
||
<span class="n">Niterations</span> <span class="o">=</span> <span class="mi">100</span>
|
||
|
||
<span class="c1"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
|
||
<span class="n">training_gradient</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">CostOLS</span><span class="p">,</span><span class="mi">2</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="nb">iter</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">Niterations</span><span class="p">):</span>
|
||
<span class="n">gradients</span> <span class="o">=</span> <span class="p">(</span><span class="mf">1.0</span><span class="o">/</span><span class="n">n</span><span class="p">)</span><span class="o">*</span><span class="n">training_gradient</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">X</span><span class="p">,</span> <span class="n">theta</span><span class="p">)</span>
|
||
<span class="n">theta</span> <span class="o">-=</span> <span class="n">eta</span><span class="o">*</span><span class="n">gradients</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"theta from own gd"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
|
||
|
||
<span class="n">n_epochs</span> <span class="o">=</span> <span class="mi">50</span>
|
||
<span class="n">M</span> <span class="o">=</span> <span class="mi">5</span> <span class="c1">#size of each minibatch</span>
|
||
<span class="n">m</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">n</span><span class="o">/</span><span class="n">M</span><span class="p">)</span> <span class="c1">#number of minibatches</span>
|
||
<span class="n">t0</span><span class="p">,</span> <span class="n">t1</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">50</span>
|
||
<span class="k">def</span> <span class="nf">learning_schedule</span><span class="p">(</span><span class="n">t</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">t0</span><span class="o">/</span><span class="p">(</span><span class="n">t</span><span class="o">+</span><span class="n">t1</span><span class="p">)</span>
|
||
|
||
<span class="n">theta</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
|
||
<span class="n">change</span> <span class="o">=</span> <span class="mf">0.0</span>
|
||
<span class="n">delta_momentum</span> <span class="o">=</span> <span class="mf">0.3</span>
|
||
|
||
<span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_epochs</span><span class="p">):</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">m</span><span class="p">):</span>
|
||
<span class="n">random_index</span> <span class="o">=</span> <span class="n">M</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="n">m</span><span class="p">)</span>
|
||
<span class="n">xi</span> <span class="o">=</span> <span class="n">X</span><span class="p">[</span><span class="n">random_index</span><span class="p">:</span><span class="n">random_index</span><span class="o">+</span><span class="n">M</span><span class="p">]</span>
|
||
<span class="n">yi</span> <span class="o">=</span> <span class="n">y</span><span class="p">[</span><span class="n">random_index</span><span class="p">:</span><span class="n">random_index</span><span class="o">+</span><span class="n">M</span><span class="p">]</span>
|
||
<span class="n">gradients</span> <span class="o">=</span> <span class="p">(</span><span class="mf">1.0</span><span class="o">/</span><span class="n">M</span><span class="p">)</span><span class="o">*</span><span class="n">training_gradient</span><span class="p">(</span><span class="n">yi</span><span class="p">,</span> <span class="n">xi</span><span class="p">,</span> <span class="n">theta</span><span class="p">)</span>
|
||
<span class="n">eta</span> <span class="o">=</span> <span class="n">learning_schedule</span><span class="p">(</span><span class="n">epoch</span><span class="o">*</span><span class="n">m</span><span class="o">+</span><span class="n">i</span><span class="p">)</span>
|
||
<span class="c1"># calculate update</span>
|
||
<span class="n">new_change</span> <span class="o">=</span> <span class="n">eta</span><span class="o">*</span><span class="n">gradients</span><span class="o">+</span><span class="n">delta_momentum</span><span class="o">*</span><span class="n">change</span>
|
||
<span class="c1"># take a step</span>
|
||
<span class="n">theta</span> <span class="o">-=</span> <span class="n">new_change</span>
|
||
<span class="c1"># save the change</span>
|
||
<span class="n">change</span> <span class="o">=</span> <span class="n">new_change</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"theta from own sdg with momentum"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
|
||
[[4.34039341]
|
||
[2.74045887]]
|
||
Eigenvalues of Hessian Matrix:[0.31262618 4.15129596]
|
||
theta from own gd
|
||
[[4.340226 ]
|
||
[2.74060714]]
|
||
theta from own sdg with momentum
|
||
[[4.30982672]
|
||
[2.69332574]]
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="adagrad-algorithm-taken-from-goodfellow-et-al">
|
||
<h2>AdaGrad algorithm, taken from <a class="reference external" href="https://www.deeplearningbook.org/contents/optimization.html">Goodfellow et al</a><a class="headerlink" href="#adagrad-algorithm-taken-from-goodfellow-et-al" title="Permalink to this headline">¶</a></h2>
|
||
<!-- dom:FIGURE: [figures/adagrad.png, width=600 frac=0.8] -->
|
||
<!-- begin figure -->
|
||
<p><img src="figures/adagrad.png" width="600"><p style="font-size: 0.9em"><i>Figure 1: </i></p></p>
|
||
<!-- end figure --></div>
|
||
<div class="section" id="similar-second-order-function-now-problem-but-now-with-adagrad">
|
||
<h2>Similar (second order function now) problem but now with AdaGrad<a class="headerlink" href="#similar-second-order-function-now-problem-but-now-with-adagrad" title="Permalink to this headline">¶</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent</span>
|
||
<span class="c1"># OLS example</span>
|
||
<span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span><span class="p">,</span> <span class="n">seed</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">autograd.numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">autograd</span> <span class="kn">import</span> <span class="n">grad</span>
|
||
|
||
<span class="c1"># Note change from previous example</span>
|
||
<span class="k">def</span> <span class="nf">CostOLS</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">X</span><span class="p">,</span><span class="n">theta</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y</span><span class="o">-</span><span class="n">X</span> <span class="o">@</span> <span class="n">theta</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span>
|
||
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">1000</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mf">2.0</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span> <span class="o">+</span><span class="mi">4</span><span class="o">*</span><span class="n">x</span><span class="o">*</span><span class="n">x</span>
|
||
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">c_</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)),</span> <span class="n">x</span><span class="p">,</span> <span class="n">x</span><span class="o">*</span><span class="n">x</span><span class="p">]</span>
|
||
<span class="n">XT_X</span> <span class="o">=</span> <span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X</span>
|
||
<span class="n">theta_linreg</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">pinv</span><span class="p">(</span><span class="n">XT_X</span><span class="p">)</span> <span class="o">@</span> <span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">y</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Own inversion"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta_linreg</span><span class="p">)</span>
|
||
|
||
|
||
<span class="c1"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
|
||
<span class="n">training_gradient</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">CostOLS</span><span class="p">,</span><span class="mi">2</span><span class="p">)</span>
|
||
<span class="c1"># Define parameters for Stochastic Gradient Descent</span>
|
||
<span class="n">n_epochs</span> <span class="o">=</span> <span class="mi">50</span>
|
||
<span class="n">M</span> <span class="o">=</span> <span class="mi">5</span> <span class="c1">#size of each minibatch</span>
|
||
<span class="n">m</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">n</span><span class="o">/</span><span class="n">M</span><span class="p">)</span> <span class="c1">#number of minibatches</span>
|
||
<span class="c1"># Guess for unknown parameters theta</span>
|
||
<span class="n">theta</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Value for learning rate</span>
|
||
<span class="n">eta</span> <span class="o">=</span> <span class="mf">0.01</span>
|
||
<span class="c1"># Including AdaGrad parameter to avoid possible division by zero</span>
|
||
<span class="n">delta</span> <span class="o">=</span> <span class="mf">1e-8</span>
|
||
<span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_epochs</span><span class="p">):</span>
|
||
<span class="n">Giter</span> <span class="o">=</span> <span class="mf">0.0</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">m</span><span class="p">):</span>
|
||
<span class="n">random_index</span> <span class="o">=</span> <span class="n">M</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="n">m</span><span class="p">)</span>
|
||
<span class="n">xi</span> <span class="o">=</span> <span class="n">X</span><span class="p">[</span><span class="n">random_index</span><span class="p">:</span><span class="n">random_index</span><span class="o">+</span><span class="n">M</span><span class="p">]</span>
|
||
<span class="n">yi</span> <span class="o">=</span> <span class="n">y</span><span class="p">[</span><span class="n">random_index</span><span class="p">:</span><span class="n">random_index</span><span class="o">+</span><span class="n">M</span><span class="p">]</span>
|
||
<span class="n">gradients</span> <span class="o">=</span> <span class="p">(</span><span class="mf">1.0</span><span class="o">/</span><span class="n">M</span><span class="p">)</span><span class="o">*</span><span class="n">training_gradient</span><span class="p">(</span><span class="n">yi</span><span class="p">,</span> <span class="n">xi</span><span class="p">,</span> <span class="n">theta</span><span class="p">)</span>
|
||
<span class="n">Giter</span> <span class="o">+=</span> <span class="n">gradients</span><span class="o">*</span><span class="n">gradients</span>
|
||
<span class="n">update</span> <span class="o">=</span> <span class="n">gradients</span><span class="o">*</span><span class="n">eta</span><span class="o">/</span><span class="p">(</span><span class="n">delta</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">Giter</span><span class="p">))</span>
|
||
<span class="n">theta</span> <span class="o">-=</span> <span class="n">update</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"theta from own AdaGrad"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
|
||
[[2.]
|
||
[3.]
|
||
[4.]]
|
||
</pre></div>
|
||
</div>
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>theta from own AdaGrad
|
||
[[2.0009282 ]
|
||
[2.99447213]
|
||
[4.00526145]]
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Running this code we note an almost perfect agreement with the results from matrix inversion.</p>
|
||
</div>
|
||
<div class="section" id="rmsprop-algorithm-taken-from-goodfellow-et-al">
|
||
<h2>RMSProp algorithm, taken from <a class="reference external" href="https://www.deeplearningbook.org/contents/optimization.html">Goodfellow et al</a><a class="headerlink" href="#rmsprop-algorithm-taken-from-goodfellow-et-al" title="Permalink to this headline">¶</a></h2>
|
||
<!-- dom:FIGURE: [figures/rmsprop.png, width=600 frac=0.8] -->
|
||
<!-- begin figure -->
|
||
<p><img src="figures/rmsprop.png" width="600"><p style="font-size: 0.9em"><i>Figure 1: </i></p></p>
|
||
<!-- end figure --></div>
|
||
<div class="section" id="rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent">
|
||
<h2>RMSprop for adaptive learning rate with Stochastic Gradient Descent<a class="headerlink" href="#rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent" title="Permalink to this headline">¶</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent</span>
|
||
<span class="c1"># OLS example</span>
|
||
<span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span><span class="p">,</span> <span class="n">seed</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">autograd.numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">autograd</span> <span class="kn">import</span> <span class="n">grad</span>
|
||
|
||
<span class="c1"># Note change from previous example</span>
|
||
<span class="k">def</span> <span class="nf">CostOLS</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">X</span><span class="p">,</span><span class="n">theta</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y</span><span class="o">-</span><span class="n">X</span> <span class="o">@</span> <span class="n">theta</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span>
|
||
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">1000</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mf">2.0</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span> <span class="o">+</span><span class="mi">4</span><span class="o">*</span><span class="n">x</span><span class="o">*</span><span class="n">x</span><span class="c1"># +np.random.randn(n,1)</span>
|
||
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">c_</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)),</span> <span class="n">x</span><span class="p">,</span> <span class="n">x</span><span class="o">*</span><span class="n">x</span><span class="p">]</span>
|
||
<span class="n">XT_X</span> <span class="o">=</span> <span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X</span>
|
||
<span class="n">theta_linreg</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">pinv</span><span class="p">(</span><span class="n">XT_X</span><span class="p">)</span> <span class="o">@</span> <span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">y</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Own inversion"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta_linreg</span><span class="p">)</span>
|
||
|
||
|
||
<span class="c1"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
|
||
<span class="n">training_gradient</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">CostOLS</span><span class="p">,</span><span class="mi">2</span><span class="p">)</span>
|
||
<span class="c1"># Define parameters for Stochastic Gradient Descent</span>
|
||
<span class="n">n_epochs</span> <span class="o">=</span> <span class="mi">50</span>
|
||
<span class="n">M</span> <span class="o">=</span> <span class="mi">5</span> <span class="c1">#size of each minibatch</span>
|
||
<span class="n">m</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">n</span><span class="o">/</span><span class="n">M</span><span class="p">)</span> <span class="c1">#number of minibatches</span>
|
||
<span class="c1"># Guess for unknown parameters theta</span>
|
||
<span class="n">theta</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Value for learning rate</span>
|
||
<span class="n">eta</span> <span class="o">=</span> <span class="mf">0.01</span>
|
||
<span class="c1"># Value for parameter rho</span>
|
||
<span class="n">rho</span> <span class="o">=</span> <span class="mf">0.99</span>
|
||
<span class="c1"># Including AdaGrad parameter to avoid possible division by zero</span>
|
||
<span class="n">delta</span> <span class="o">=</span> <span class="mf">1e-8</span>
|
||
<span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_epochs</span><span class="p">):</span>
|
||
<span class="n">Giter</span> <span class="o">=</span> <span class="mf">0.0</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">m</span><span class="p">):</span>
|
||
<span class="n">random_index</span> <span class="o">=</span> <span class="n">M</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="n">m</span><span class="p">)</span>
|
||
<span class="n">xi</span> <span class="o">=</span> <span class="n">X</span><span class="p">[</span><span class="n">random_index</span><span class="p">:</span><span class="n">random_index</span><span class="o">+</span><span class="n">M</span><span class="p">]</span>
|
||
<span class="n">yi</span> <span class="o">=</span> <span class="n">y</span><span class="p">[</span><span class="n">random_index</span><span class="p">:</span><span class="n">random_index</span><span class="o">+</span><span class="n">M</span><span class="p">]</span>
|
||
<span class="n">gradients</span> <span class="o">=</span> <span class="p">(</span><span class="mf">1.0</span><span class="o">/</span><span class="n">M</span><span class="p">)</span><span class="o">*</span><span class="n">training_gradient</span><span class="p">(</span><span class="n">yi</span><span class="p">,</span> <span class="n">xi</span><span class="p">,</span> <span class="n">theta</span><span class="p">)</span>
|
||
<span class="c1"># Accumulated gradient</span>
|
||
<span class="c1"># Scaling with rho the new and the previous results</span>
|
||
<span class="n">Giter</span> <span class="o">=</span> <span class="p">(</span><span class="n">rho</span><span class="o">*</span><span class="n">Giter</span><span class="o">+</span><span class="p">(</span><span class="mi">1</span><span class="o">-</span><span class="n">rho</span><span class="p">)</span><span class="o">*</span><span class="n">gradients</span><span class="o">*</span><span class="n">gradients</span><span class="p">)</span>
|
||
<span class="c1"># Taking the diagonal only and inverting</span>
|
||
<span class="n">update</span> <span class="o">=</span> <span class="n">gradients</span><span class="o">*</span><span class="n">eta</span><span class="o">/</span><span class="p">(</span><span class="n">delta</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">Giter</span><span class="p">))</span>
|
||
<span class="c1"># Hadamard product</span>
|
||
<span class="n">theta</span> <span class="o">-=</span> <span class="n">update</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"theta from own RMSprop"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
|
||
[[2.]
|
||
[3.]
|
||
[4.]]
|
||
</pre></div>
|
||
</div>
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>theta from own RMSprop
|
||
[[1.99998409]
|
||
[2.99993515]
|
||
[4.00003694]]
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="adam-algorithm-taken-from-goodfellow-et-al">
|
||
<h2>ADAM algorithm, taken from <a class="reference external" href="https://www.deeplearningbook.org/contents/optimization.html">Goodfellow et al</a><a class="headerlink" href="#adam-algorithm-taken-from-goodfellow-et-al" title="Permalink to this headline">¶</a></h2>
|
||
<!-- dom:FIGURE: [figures/adam.png, width=600 frac=0.8] -->
|
||
<!-- begin figure -->
|
||
<p><img src="figures/adam.png" width="600"><p style="font-size: 0.9em"><i>Figure 1: </i></p></p>
|
||
<!-- end figure --></div>
|
||
<div class="section" id="and-finally-adam">
|
||
<h2>And finally <a class="reference external" href="https://arxiv.org/pdf/1412.6980.pdf">ADAM</a><a class="headerlink" href="#and-finally-adam" title="Permalink to this headline">¶</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent</span>
|
||
<span class="c1"># OLS example</span>
|
||
<span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span><span class="p">,</span> <span class="n">seed</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">autograd.numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">autograd</span> <span class="kn">import</span> <span class="n">grad</span>
|
||
|
||
<span class="c1"># Note change from previous example</span>
|
||
<span class="k">def</span> <span class="nf">CostOLS</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">X</span><span class="p">,</span><span class="n">theta</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y</span><span class="o">-</span><span class="n">X</span> <span class="o">@</span> <span class="n">theta</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span>
|
||
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">1000</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mf">2.0</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span> <span class="o">+</span><span class="mi">4</span><span class="o">*</span><span class="n">x</span><span class="o">*</span><span class="n">x</span><span class="c1"># +np.random.randn(n,1)</span>
|
||
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">c_</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="n">n</span><span class="p">,</span><span class="mi">1</span><span class="p">)),</span> <span class="n">x</span><span class="p">,</span> <span class="n">x</span><span class="o">*</span><span class="n">x</span><span class="p">]</span>
|
||
<span class="n">XT_X</span> <span class="o">=</span> <span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X</span>
|
||
<span class="n">theta_linreg</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">pinv</span><span class="p">(</span><span class="n">XT_X</span><span class="p">)</span> <span class="o">@</span> <span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">y</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Own inversion"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta_linreg</span><span class="p">)</span>
|
||
|
||
|
||
<span class="c1"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
|
||
<span class="n">training_gradient</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">CostOLS</span><span class="p">,</span><span class="mi">2</span><span class="p">)</span>
|
||
<span class="c1"># Define parameters for Stochastic Gradient Descent</span>
|
||
<span class="n">n_epochs</span> <span class="o">=</span> <span class="mi">50</span>
|
||
<span class="n">M</span> <span class="o">=</span> <span class="mi">5</span> <span class="c1">#size of each minibatch</span>
|
||
<span class="n">m</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">n</span><span class="o">/</span><span class="n">M</span><span class="p">)</span> <span class="c1">#number of minibatches</span>
|
||
<span class="c1"># Guess for unknown parameters theta</span>
|
||
<span class="n">theta</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Value for learning rate</span>
|
||
<span class="n">eta</span> <span class="o">=</span> <span class="mf">0.01</span>
|
||
<span class="c1"># Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980</span>
|
||
<span class="n">beta1</span> <span class="o">=</span> <span class="mf">0.9</span>
|
||
<span class="n">beta2</span> <span class="o">=</span> <span class="mf">0.999</span>
|
||
<span class="c1"># Including AdaGrad parameter to avoid possible division by zero</span>
|
||
<span class="n">delta</span> <span class="o">=</span> <span class="mf">1e-7</span>
|
||
<span class="nb">iter</span> <span class="o">=</span> <span class="mi">0</span>
|
||
<span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_epochs</span><span class="p">):</span>
|
||
<span class="n">first_moment</span> <span class="o">=</span> <span class="mf">0.0</span>
|
||
<span class="n">second_moment</span> <span class="o">=</span> <span class="mf">0.0</span>
|
||
<span class="nb">iter</span> <span class="o">+=</span> <span class="mi">1</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">m</span><span class="p">):</span>
|
||
<span class="n">random_index</span> <span class="o">=</span> <span class="n">M</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="n">m</span><span class="p">)</span>
|
||
<span class="n">xi</span> <span class="o">=</span> <span class="n">X</span><span class="p">[</span><span class="n">random_index</span><span class="p">:</span><span class="n">random_index</span><span class="o">+</span><span class="n">M</span><span class="p">]</span>
|
||
<span class="n">yi</span> <span class="o">=</span> <span class="n">y</span><span class="p">[</span><span class="n">random_index</span><span class="p">:</span><span class="n">random_index</span><span class="o">+</span><span class="n">M</span><span class="p">]</span>
|
||
<span class="n">gradients</span> <span class="o">=</span> <span class="p">(</span><span class="mf">1.0</span><span class="o">/</span><span class="n">M</span><span class="p">)</span><span class="o">*</span><span class="n">training_gradient</span><span class="p">(</span><span class="n">yi</span><span class="p">,</span> <span class="n">xi</span><span class="p">,</span> <span class="n">theta</span><span class="p">)</span>
|
||
<span class="c1"># Computing moments first</span>
|
||
<span class="n">first_moment</span> <span class="o">=</span> <span class="n">beta1</span><span class="o">*</span><span class="n">first_moment</span> <span class="o">+</span> <span class="p">(</span><span class="mi">1</span><span class="o">-</span><span class="n">beta1</span><span class="p">)</span><span class="o">*</span><span class="n">gradients</span>
|
||
<span class="n">second_moment</span> <span class="o">=</span> <span class="n">beta2</span><span class="o">*</span><span class="n">second_moment</span><span class="o">+</span><span class="p">(</span><span class="mi">1</span><span class="o">-</span><span class="n">beta2</span><span class="p">)</span><span class="o">*</span><span class="n">gradients</span><span class="o">*</span><span class="n">gradients</span>
|
||
<span class="n">first_term</span> <span class="o">=</span> <span class="n">first_moment</span><span class="o">/</span><span class="p">(</span><span class="mf">1.0</span><span class="o">-</span><span class="n">beta1</span><span class="o">**</span><span class="nb">iter</span><span class="p">)</span>
|
||
<span class="n">second_term</span> <span class="o">=</span> <span class="n">second_moment</span><span class="o">/</span><span class="p">(</span><span class="mf">1.0</span><span class="o">-</span><span class="n">beta2</span><span class="o">**</span><span class="nb">iter</span><span class="p">)</span>
|
||
<span class="c1"># Scaling with rho the new and the previous results</span>
|
||
<span class="n">update</span> <span class="o">=</span> <span class="n">eta</span><span class="o">*</span><span class="n">first_term</span><span class="o">/</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">second_term</span><span class="p">)</span><span class="o">+</span><span class="n">delta</span><span class="p">)</span>
|
||
<span class="n">theta</span> <span class="o">-=</span> <span class="n">update</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"theta from own ADAM"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
|
||
[[2.]
|
||
[3.]
|
||
[4.]]
|
||
</pre></div>
|
||
</div>
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>theta from own ADAM
|
||
[[2.00011851]
|
||
[2.99937234]
|
||
[4.00068042]]
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="introducing-jax">
|
||
<h2>Introducing <a class="reference external" href="https://jax.readthedocs.io/en/latest/">JAX</a><a class="headerlink" href="#introducing-jax" title="Permalink to this headline">¶</a></h2>
|
||
<p>Presently, instead of using <strong>autograd</strong>, we recommend using <a class="reference external" href="https://jax.readthedocs.io/en/latest/">JAX</a></p>
|
||
<p><strong>JAX</strong> is Autograd and <a class="reference external" href="https://www.tensorflow.org/xla">XLA (Accelerated Linear Algebra))</a>,
|
||
brought together for high-performance numerical computing and machine learning research.
|
||
It provides composable transformations of Python+NumPy programs: differentiate, vectorize, parallelize, Just-In-Time compile to GPU/TPU, and more.</p>
|
||
<div class="section" id="getting-started-with-jax-note-the-way-we-import-numpy">
|
||
<h3>Getting started with Jax, note the way we import numpy<a class="headerlink" href="#getting-started-with-jax-note-the-way-we-import-numpy" title="Permalink to this headline">¶</a></h3>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">jax</span>
|
||
<span class="kn">import</span> <span class="nn">jax.numpy</span> <span class="k">as</span> <span class="nn">jnp</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
|
||
<span class="kn">from</span> <span class="nn">jax</span> <span class="kn">import</span> <span class="n">grad</span> <span class="k">as</span> <span class="n">jax_grad</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="a-warm-up-example">
|
||
<h3>A warm-up example<a class="headerlink" href="#a-warm-up-example" title="Permalink to this headline">¶</a></h3>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">function</span><span class="p">(</span><span class="n">x</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">x</span><span class="o">**</span><span class="mi">2</span>
|
||
|
||
<span class="k">def</span> <span class="nf">analytical_gradient</span><span class="p">(</span><span class="n">x</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="mi">2</span><span class="o">*</span><span class="n">x</span>
|
||
|
||
<span class="k">def</span> <span class="nf">gradient_descent</span><span class="p">(</span><span class="n">starting_point</span><span class="p">,</span> <span class="n">learning_rate</span><span class="p">,</span> <span class="n">num_iterations</span><span class="p">,</span> <span class="n">solver</span><span class="o">=</span><span class="s2">"analytical"</span><span class="p">):</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">starting_point</span>
|
||
<span class="n">trajectory_x</span> <span class="o">=</span> <span class="p">[</span><span class="n">x</span><span class="p">]</span>
|
||
<span class="n">trajectory_y</span> <span class="o">=</span> <span class="p">[</span><span class="n">function</span><span class="p">(</span><span class="n">x</span><span class="p">)]</span>
|
||
|
||
<span class="k">if</span> <span class="n">solver</span> <span class="o">==</span> <span class="s2">"analytical"</span><span class="p">:</span>
|
||
<span class="n">grad</span> <span class="o">=</span> <span class="n">analytical_gradient</span>
|
||
<span class="k">elif</span> <span class="n">solver</span> <span class="o">==</span> <span class="s2">"jax"</span><span class="p">:</span>
|
||
<span class="n">grad</span> <span class="o">=</span> <span class="n">jax_grad</span><span class="p">(</span><span class="n">function</span><span class="p">)</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">jnp</span><span class="o">.</span><span class="n">float64</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="n">learning_rate</span> <span class="o">=</span> <span class="n">jnp</span><span class="o">.</span><span class="n">float64</span><span class="p">(</span><span class="n">learning_rate</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_iterations</span><span class="p">):</span>
|
||
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">x</span> <span class="o">-</span> <span class="n">learning_rate</span> <span class="o">*</span> <span class="n">grad</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="n">trajectory_x</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="n">trajectory_y</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">function</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
|
||
|
||
<span class="k">return</span> <span class="n">trajectory_x</span><span class="p">,</span> <span class="n">trajectory_y</span>
|
||
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="o">-</span><span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">100</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">function</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">label</span><span class="o">=</span><span class="s2">"f(x)"</span><span class="p">)</span>
|
||
|
||
<span class="n">descent_x</span><span class="p">,</span> <span class="n">descent_y</span> <span class="o">=</span> <span class="n">gradient_descent</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="n">solver</span><span class="o">=</span><span class="s2">"analytical"</span><span class="p">)</span>
|
||
<span class="n">jax_descend_x</span><span class="p">,</span> <span class="n">jax_descend_y</span> <span class="o">=</span> <span class="n">gradient_descent</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="n">solver</span><span class="o">=</span><span class="s2">"jax"</span><span class="p">)</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">descent_x</span><span class="p">,</span> <span class="n">descent_y</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">"Gradient descent"</span><span class="p">,</span> <span class="n">marker</span><span class="o">=</span><span class="s2">"o"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">jax_descend_x</span><span class="p">,</span> <span class="n">jax_descend_y</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">"JAX"</span><span class="p">,</span> <span class="n">marker</span><span class="o">=</span><span class="s2">"x"</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/jax/_src/numpy/lax_numpy.py:173: UserWarning: Explicitly requested dtype float64 requested in asarray is not available, and will be truncated to dtype float32. To enable more dtypes, set the jax_enable_x64 configuration option or the JAX_ENABLE_X64 shell environment variable. See https://github.com/google/jax#current-gotchas for more.
|
||
return asarray(x, dtype=self.dtype)
|
||
</pre></div>
|
||
</div>
|
||
<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[<matplotlib.lines.Line2D at 0x125606520>]
|
||
</pre></div>
|
||
</div>
|
||
<img alt="_images/exercisesweek41_39_2.png" src="_images/exercisesweek41_39_2.png" />
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="a-more-advanced-example">
|
||
<h3>A more advanced example<a class="headerlink" href="#a-more-advanced-example" title="Permalink to this headline">¶</a></h3>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">backend</span> <span class="o">=</span> <span class="n">np</span>
|
||
|
||
<span class="k">def</span> <span class="nf">function</span><span class="p">(</span><span class="n">x</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">x</span><span class="o">*</span><span class="n">backend</span><span class="o">.</span><span class="n">sin</span><span class="p">(</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">analytical_gradient</span><span class="p">(</span><span class="n">x</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">backend</span><span class="o">.</span><span class="n">sin</span><span class="p">(</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">+</span> <span class="mi">2</span><span class="o">*</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span><span class="o">*</span><span class="n">backend</span><span class="o">.</span><span class="n">cos</span><span class="p">(</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
|
||
|
||
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="o">-</span><span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">100</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">function</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">label</span><span class="o">=</span><span class="s2">"f(x)"</span><span class="p">)</span>
|
||
|
||
<span class="n">descent_x</span><span class="p">,</span> <span class="n">descent_y</span> <span class="o">=</span> <span class="n">gradient_descent</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mf">0.01</span><span class="p">,</span> <span class="mi">300</span><span class="p">,</span> <span class="n">solver</span><span class="o">=</span><span class="s2">"analytical"</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Change the backend to JAX</span>
|
||
<span class="n">backend</span> <span class="o">=</span> <span class="n">jnp</span>
|
||
<span class="n">jax_descend_x</span><span class="p">,</span> <span class="n">jax_descend_y</span> <span class="o">=</span> <span class="n">gradient_descent</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mf">0.01</span><span class="p">,</span> <span class="mi">300</span><span class="p">,</span> <span class="n">solver</span><span class="o">=</span><span class="s2">"jax"</span><span class="p">)</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">scatter</span><span class="p">(</span><span class="n">descent_x</span><span class="p">,</span> <span class="n">descent_y</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">"Gradient descent"</span><span class="p">,</span> <span class="n">marker</span><span class="o">=</span><span class="s2">"v"</span><span class="p">,</span> <span class="n">s</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="s2">"red"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">scatter</span><span class="p">(</span><span class="n">jax_descend_x</span><span class="p">,</span> <span class="n">jax_descend_y</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">"JAX"</span><span class="p">,</span> <span class="n">marker</span><span class="o">=</span><span class="s2">"x"</span><span class="p">,</span> <span class="n">s</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="s2">"black"</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><matplotlib.collections.PathCollection at 0x1253cc610>
|
||
</pre></div>
|
||
</div>
|
||
<img alt="_images/exercisesweek41_41_1.png" src="_images/exercisesweek41_41_1.png" />
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
kernelName: "python3",
|
||
path: "./."
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</div>
|
||
|
||
|
||
<!-- Previous / next buttons -->
|
||
<div class='prev-next-area'>
|
||
<a class='left-prev' id="prev-link" href="week40.html" title="previous page">
|
||
<i class="fas fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title">Week 40: Gradient descent methods (continued) and start Neural networks</p>
|
||
</div>
|
||
</a>
|
||
<a class='right-next' id="next-link" href="week41.html" title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title">Week 41 Neural networks and constructing a neural network code</p>
|
||
</div>
|
||
<i class="fas fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
<footer class="footer">
|
||
<p>
|
||
|
||
By Morten Hjorth-Jensen<br/>
|
||
|
||
© Copyright 2021.<br/>
|
||
</p>
|
||
</footer>
|
||
</main>
|
||
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<script src="_static/js/index.be7d3bbb2ef33a8344ce.js"></script>
|
||
|
||
</body>
|
||
</html> |