Morten Hjorth-Jensen
1168 lines
81 KiB
HTML
1168 lines
81 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
<html>
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<title>Exercises week 43 — Applied Data Analysis and Machine Learning</title>
|
||
|
||
<link href="_static/css/theme.css" rel="stylesheet">
|
||
<link href="_static/css/index.ff1ffe594081f20da1ef19478df9384b.css" rel="stylesheet">
|
||
|
||
|
||
<link rel="stylesheet"
|
||
href="_static/vendor/fontawesome/5.13.0/css/all.min.css">
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin
|
||
href="_static/vendor/fontawesome/5.13.0/webfonts/fa-solid-900.woff2">
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin
|
||
href="_static/vendor/fontawesome/5.13.0/webfonts/fa-brands-400.woff2">
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-book-theme.css?digest=c3fdc42140077d1ad13ad2f1588a4309" />
|
||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/mystnb.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/panels-main.c949a650a448cc0ae9fd3441c0e17fb0.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/panels-variables.06eb56fa6e07937060861dad626602ad.css" />
|
||
|
||
<link rel="preload" as="script" href="_static/js/index.be7d3bbb2ef33a8344ce.js">
|
||
|
||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||
<script src="_static/jquery.js"></script>
|
||
<script src="_static/underscore.js"></script>
|
||
<script src="_static/doctools.js"></script>
|
||
<script src="_static/clipboard.min.js"></script>
|
||
<script src="_static/copybutton.js"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="_static/togglebutton.js"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown, .tag_hide_input div.cell_input, .tag_hide-input div.cell_input, .tag_hide_output div.cell_output, .tag_hide-output div.cell_output, .tag_hide_cell.cell, .tag_hide-cell.cell';</script>
|
||
<script src="_static/sphinx-book-theme.d59cb220de22ca1c485ebbdc042f0030.js"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"
|
||
const thebe_selector = ".thebe,.cell"
|
||
const thebe_selector_input = "pre"
|
||
const thebe_selector_output = ".output, .cell_output"
|
||
</script>
|
||
<script async="async" src="_static/sphinx-thebe.js"></script>
|
||
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
||
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="Week 44, Convolutional Neural Networks (CNN)" href="week44.html" />
|
||
<link rel="prev" title="Week 43: Deep Learning: Constructing a Neural Network code and solving differential equations" href="week43.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
<meta name="docsearch:language" content="None">
|
||
|
||
|
||
<!-- Google Analytics -->
|
||
|
||
</head>
|
||
<body data-spy="scroll" data-target="#bd-toc-nav" data-offset="80">
|
||
|
||
<div class="container-fluid" id="banner"></div>
|
||
|
||
|
||
|
||
<div class="container-xl">
|
||
<div class="row">
|
||
|
||
<div class="col-12 col-md-3 bd-sidebar site-navigation show" id="site-navigation">
|
||
|
||
<div class="navbar-brand-box">
|
||
<a class="navbar-brand text-wrap" href="index.html">
|
||
|
||
<!-- `logo` is deprecated in Sphinx 4.0, so remove this when we stop supporting 3 -->
|
||
|
||
|
||
|
||
<img src="_static/logo.png" class="logo" alt="logo">
|
||
|
||
|
||
<h1 class="site-logo" id="site-title">Applied Data Analysis and Machine Learning</h1>
|
||
|
||
</a>
|
||
</div><form class="bd-search d-flex align-items-center" action="search.html" method="get">
|
||
<i class="icon fas fa-search"></i>
|
||
<input type="search" class="form-control" name="q" id="search-input" placeholder="Search this book..." aria-label="Search this book..." autocomplete="off" >
|
||
</form><nav class="bd-links" id="bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item active">
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="intro.html">
|
||
Applied Data Analysis and Machine Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
About the course
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="schedule.html">
|
||
Teaching schedule with links to material
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="teachers.html">
|
||
Teachers and Grading
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="textbooks.html">
|
||
Textbooks
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Review of Statistics with Resampling Techniques and Linear Algebra
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="statistics.html">
|
||
1. Elements of Probability Theory and Statistical Data Analysis
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="linalg.html">
|
||
2. Linear Algebra, Handling of Arrays and more Python Features
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
From Regression to Support Vector Machines
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter1.html">
|
||
3. Linear Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter2.html">
|
||
4. Ridge and Lasso Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter3.html">
|
||
5. Resampling Methods
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter4.html">
|
||
6. Logistic Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapteroptimization.html">
|
||
7. Optimization, the central part of any Machine Learning algortithm
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter5.html">
|
||
8. Support Vector Machines, overarching aims
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Decision Trees, Ensemble Methods and Boosting
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter6.html">
|
||
9. Decision trees, overarching aims
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter7.html">
|
||
10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Dimensionality Reduction
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter8.html">
|
||
11. Basic ideas of the Principal Component Analysis (PCA)
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="clustering.html">
|
||
12. Clustering and Unsupervised Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Deep Learning Methods
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter9.html">
|
||
13. Neural networks
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter10.html">
|
||
14. Building a Feed Forward Neural Network
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter11.html">
|
||
15. Solving Differential Equations with Deep Learning
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter12.html">
|
||
16. Convolutional Neural Networks
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter13.html">
|
||
17. Recurrent neural networks: Overarching view
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Weekly material, notes and exercises
|
||
</span>
|
||
</p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek34.html">
|
||
Exercises week 34
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week34.html">
|
||
Week 34: Introduction to the course, Logistics and Practicalities
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek35.html">
|
||
Exercises week 35
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week35.html">
|
||
Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek36.html">
|
||
Exercises week 36
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week36.html">
|
||
Week 36: Linear Regression and Statistical interpretations
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek37.html">
|
||
Exercises week 37
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week37.html">
|
||
Week 37: Statistical interpretations and Resampling Methods
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek38.html">
|
||
Exercises week 38
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week38.html">
|
||
Week 38: Logistic Regression and Optimization
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek39.html">
|
||
Exercises week 39
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week39.html">
|
||
Week 39: Optimization and Gradient Methods
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week40.html">
|
||
Week 40: Gradient descent methods (continued) and start Neural networks
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek41.html">
|
||
Exercises week 41
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week41.html">
|
||
Week 41 Neural networks and constructing a neural network code
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek42.html">
|
||
Exercises week 42
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week42.html">
|
||
Week 42 Constructing a Neural Network code with examples
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="additionweek42.html">
|
||
Exercises Week 42: Logistic Regression and Optimization, reminders from week 38 and week 40
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week43.html">
|
||
Week 43: Deep Learning: Constructing a Neural Network code and solving differential equations
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1 current active">
|
||
<a class="current reference internal" href="#">
|
||
Exercises week 43
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week44.html">
|
||
Week 44, Convolutional Neural Networks (CNN)
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Projects
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="project1.html">
|
||
Project 1 on Machine Learning, deadline October 7 (midnight), 2024
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="project2.html">
|
||
Project 2 on Machine Learning, deadline November 4 (Midnight)
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</div>
|
||
</nav> <!-- To handle the deprecated key -->
|
||
|
||
<div class="navbar_extra_footer">
|
||
Powered by <a href="https://jupyterbook.org">Jupyter Book</a>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<main class="col py-md-3 pl-md-4 bd-content overflow-auto" role="main">
|
||
|
||
<div class="topbar container-xl fixed-top">
|
||
<div class="topbar-contents row">
|
||
<div class="col-12 col-md-3 bd-topbar-whitespace site-navigation show"></div>
|
||
<div class="col pl-md-4 topbar-main">
|
||
|
||
<button id="navbar-toggler" class="navbar-toggler ml-0" type="button" data-toggle="collapse"
|
||
data-toggle="tooltip" data-placement="bottom" data-target=".site-navigation" aria-controls="navbar-menu"
|
||
aria-expanded="true" aria-label="Toggle navigation" aria-controls="site-navigation"
|
||
title="Toggle navigation" data-toggle="tooltip" data-placement="left">
|
||
<i class="fas fa-bars"></i>
|
||
<i class="fas fa-arrow-left"></i>
|
||
<i class="fas fa-arrow-up"></i>
|
||
</button>
|
||
|
||
|
||
<div class="dropdown-buttons-trigger">
|
||
<button id="dropdown-buttons-trigger" class="btn btn-secondary topbarbtn" aria-label="Download this page"><i
|
||
class="fas fa-download"></i></button>
|
||
|
||
<div class="dropdown-buttons">
|
||
<!-- ipynb file if we had a myst markdown file -->
|
||
|
||
<!-- Download raw file -->
|
||
<a class="dropdown-buttons" href="_sources/exercisesweek43.ipynb"><button type="button"
|
||
class="btn btn-secondary topbarbtn" title="Download source file" data-toggle="tooltip"
|
||
data-placement="left">.ipynb</button></a>
|
||
<!-- Download PDF via print -->
|
||
<button type="button" id="download-print" class="btn btn-secondary topbarbtn" title="Print to PDF"
|
||
onclick="printPdf(this)" data-toggle="tooltip" data-placement="left">.pdf</button>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Source interaction buttons -->
|
||
|
||
<!-- Full screen (wrap in <a> to have style consistency -->
|
||
|
||
<a class="full-screen-button"><button type="button" class="btn btn-secondary topbarbtn" data-toggle="tooltip"
|
||
data-placement="bottom" onclick="toggleFullScreen()" aria-label="Fullscreen mode"
|
||
title="Fullscreen mode"><i
|
||
class="fas fa-expand"></i></button></a>
|
||
|
||
<!-- Launch buttons -->
|
||
|
||
</div>
|
||
|
||
<!-- Table of contents -->
|
||
<div class="d-none d-md-block col-md-2 bd-toc show noprint">
|
||
|
||
<div class="tocsection onthispage pt-5 pb-3">
|
||
<i class="fas fa-list"></i> Contents
|
||
</div>
|
||
<nav id="bd-toc-nav" aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#">
|
||
Exercises week 43
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#overarching-aims-of-the-exercises-this-week">
|
||
Overarching aims of the exercises this week
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#exercise-1-understand-the-feed-forward-pass">
|
||
Exercise 1 - Understand the feed forward pass
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#exercise-2-gradient-with-one-layer-using-autograd">
|
||
Exercise 2 - Gradient with one layer using autograd
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#exercise-3-gradient-with-one-layer-writing-backpropagation-by-hand">
|
||
Exercise 3 - Gradient with one layer writing backpropagation by hand
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#exercise-4-gradient-with-two-layers-writing-backpropagation-by-hand">
|
||
Exercise 4 - Gradient with two layers writing backpropagation by hand
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#exercise-5-gradient-with-any-number-of-layers-writing-backpropagation-by-hand">
|
||
Exercise 5 - Gradient with any number of layers writing backpropagation by hand
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#exercise-6-batched-inputs">
|
||
Exercise 6 - Batched inputs
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#exercise-7-training">
|
||
Exercise 7 - Training
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#exercise-8-optional-object-orientation">
|
||
Exercise 8 (Optional) - Object orientation
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div id="main-content" class="row">
|
||
<div class="col-12 col-md-9 pl-md-3 pr-md-0">
|
||
<!-- Table of contents that is only displayed when printing the page -->
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>Exercises week 43</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#">
|
||
Exercises week 43
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#overarching-aims-of-the-exercises-this-week">
|
||
Overarching aims of the exercises this week
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#exercise-1-understand-the-feed-forward-pass">
|
||
Exercise 1 - Understand the feed forward pass
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#exercise-2-gradient-with-one-layer-using-autograd">
|
||
Exercise 2 - Gradient with one layer using autograd
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#exercise-3-gradient-with-one-layer-writing-backpropagation-by-hand">
|
||
Exercise 3 - Gradient with one layer writing backpropagation by hand
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#exercise-4-gradient-with-two-layers-writing-backpropagation-by-hand">
|
||
Exercise 4 - Gradient with two layers writing backpropagation by hand
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#exercise-5-gradient-with-any-number-of-layers-writing-backpropagation-by-hand">
|
||
Exercise 5 - Gradient with any number of layers writing backpropagation by hand
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#exercise-6-batched-inputs">
|
||
Exercise 6 - Batched inputs
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#exercise-7-training">
|
||
Exercise 7 - Training
|
||
</a>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#exercise-8-optional-object-orientation">
|
||
Exercise 8 (Optional) - Object orientation
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div>
|
||
|
||
<div class="tex2jax_ignore mathjax_ignore section" id="exercises-week-43">
|
||
<h1>Exercises week 43<a class="headerlink" href="#exercises-week-43" title="Permalink to this headline">¶</a></h1>
|
||
<p><strong>October 18-25, 2024</strong></p>
|
||
<p>Date: <strong>Deadline is Friday October 25 at midnight</strong></p>
|
||
</div>
|
||
<div class="tex2jax_ignore mathjax_ignore section" id="overarching-aims-of-the-exercises-this-week">
|
||
<h1>Overarching aims of the exercises this week<a class="headerlink" href="#overarching-aims-of-the-exercises-this-week" title="Permalink to this headline">¶</a></h1>
|
||
<p>The aim of the exercises this week is to train the neural network you implemented last week.</p>
|
||
<p>To train neural networks, we use gradient descent, since there is no analytical expression for the optimal parameters. This means you will need to compute the gradient of the cost function wrt. the network parameters. And then you will need to implement some gradient method.</p>
|
||
<p>You will begin by computing gradients for a network with one layer, then two layers, then any number of layers. Keeping track of the shapes and doing things step by step will be very important this week.</p>
|
||
<p>We recommend that you do the exercises this week by editing and running this notebook file, as it includes some checks along the way that you have implemented the neural network correctly, and running small parts of the code at a time will be important for understanding the methods. If you have trouble running a notebook, you can run this notebook in google colab instead(<a class="reference external" href="https://colab.research.google.com/drive/1FfvbN0XlhV-lATRPyGRTtTBnJr3zNuHL#offline=true&sandboxMode=true">https://colab.research.google.com/drive/1FfvbN0XlhV-lATRPyGRTtTBnJr3zNuHL#offline=true&sandboxMode=true</a>), though we recommend that you set up VSCode and your python environment to run code like this locally.</p>
|
||
<p>First, some setup code that you will need.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">autograd.numpy</span> <span class="k">as</span> <span class="nn">np</span> <span class="c1"># We need to use this numpy wrapper to make automatic differentiation work later</span>
|
||
<span class="kn">from</span> <span class="nn">autograd</span> <span class="kn">import</span> <span class="n">grad</span><span class="p">,</span> <span class="n">elementwise_grad</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn</span> <span class="kn">import</span> <span class="n">datasets</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">accuracy_score</span>
|
||
|
||
|
||
<span class="c1"># Defining some activation functions</span>
|
||
<span class="k">def</span> <span class="nf">ReLU</span><span class="p">(</span><span class="n">z</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">where</span><span class="p">(</span><span class="n">z</span> <span class="o">></span> <span class="mi">0</span><span class="p">,</span> <span class="n">z</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||
|
||
|
||
<span class="c1"># Derivative of the ReLU function</span>
|
||
<span class="k">def</span> <span class="nf">ReLU_der</span><span class="p">(</span><span class="n">z</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">where</span><span class="p">(</span><span class="n">z</span> <span class="o">></span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">sigmoid</span><span class="p">(</span><span class="n">z</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="mi">1</span> <span class="o">/</span> <span class="p">(</span><span class="mi">1</span> <span class="o">+</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="n">z</span><span class="p">))</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">mse</span><span class="p">(</span><span class="n">predict</span><span class="p">,</span> <span class="n">target</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">((</span><span class="n">predict</span> <span class="o">-</span> <span class="n">target</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="tex2jax_ignore mathjax_ignore section" id="exercise-1-understand-the-feed-forward-pass">
|
||
<h1>Exercise 1 - Understand the feed forward pass<a class="headerlink" href="#exercise-1-understand-the-feed-forward-pass" title="Permalink to this headline">¶</a></h1>
|
||
<p><strong>a)</strong> Complete last weeks’ mandatory exercises if you haven’t already.</p>
|
||
</div>
|
||
<div class="tex2jax_ignore mathjax_ignore section" id="exercise-2-gradient-with-one-layer-using-autograd">
|
||
<h1>Exercise 2 - Gradient with one layer using autograd<a class="headerlink" href="#exercise-2-gradient-with-one-layer-using-autograd" title="Permalink to this headline">¶</a></h1>
|
||
<p>For the first few exercises, we will not use batched inputs. Only a single input vector is passed through the layer at a time.</p>
|
||
<p>In this exercise you will compute the gradient of a single layer. You only need to change the code in the cells right below an exercise, the rest works out of the box. Feel free to make changes and see how stuff works though!</p>
|
||
<p><strong>a)</strong> If the weights and bias of a layer has shapes (10, 4) and (10), what will the shapes of the gradients of the cost function wrt. these weights and this bias be?</p>
|
||
<p><strong>b)</strong> Complete the feed_forward_one_layer function. It should use the sigmoid activation function. Also define the weigth and bias with the correct shapes.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">feed_forward_one_layer</span><span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
|
||
<span class="n">z</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="k">return</span> <span class="n">a</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">cost_one_layer</span><span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">target</span><span class="p">):</span>
|
||
<span class="n">predict</span> <span class="o">=</span> <span class="n">feed_forward_one_layer</span><span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">mse</span><span class="p">(</span><span class="n">predict</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
|
||
|
||
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
|
||
<span class="n">target</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span>
|
||
|
||
<span class="n">W</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">b</span> <span class="o">=</span> <span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>c)</strong> Compute the gradient of the cost function wrt. the weigth and bias by running the cell below. You will not need to change anything, just make sure it runs by defining things correctly in the cell above. This code uses the autograd package which uses backprogagation to compute the gradient!</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">autograd_one_layer</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">cost_one_layer</span><span class="p">,</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span>
|
||
<span class="n">W_g</span><span class="p">,</span> <span class="n">b_g</span> <span class="o">=</span> <span class="n">autograd_one_layer</span><span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">W_g</span><span class="p">,</span> <span class="n">b_g</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output traceback highlight-ipythontb notranslate"><div class="highlight"><pre><span></span><span class="gt">---------------------------------------------------------------------------</span>
|
||
<span class="ne">KeyError</span><span class="g g-Whitespace"> </span>Traceback (most recent call last)
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/tracer.py:118,</span> in <span class="ni">new_box</span><span class="nt">(value, trace, node)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">117</span> <span class="k">try</span><span class="p">:</span>
|
||
<span class="ne">--> </span><span class="mi">118</span> <span class="k">return</span> <span class="n">box_type_mappings</span><span class="p">[</span><span class="nb">type</span><span class="p">(</span><span class="n">value</span><span class="p">)](</span><span class="n">value</span><span class="p">,</span> <span class="n">trace</span><span class="p">,</span> <span class="n">node</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">119</span> <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
|
||
|
||
<span class="ne">KeyError</span>: <class 'ellipsis'>
|
||
|
||
<span class="n">During</span> <span class="n">handling</span> <span class="n">of</span> <span class="n">the</span> <span class="n">above</span> <span class="n">exception</span><span class="p">,</span> <span class="n">another</span> <span class="n">exception</span> <span class="n">occurred</span><span class="p">:</span>
|
||
|
||
<span class="ne">TypeError</span><span class="g g-Whitespace"> </span>Traceback (most recent call last)
|
||
<span class="n">Cell</span> <span class="n">In</span><span class="p">[</span><span class="mi">3</span><span class="p">],</span> <span class="n">line</span> <span class="mi">2</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1</span> <span class="n">autograd_one_layer</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">cost_one_layer</span><span class="p">,</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span>
|
||
<span class="ne">----> </span><span class="mi">2</span> <span class="n">W_g</span><span class="p">,</span> <span class="n">b_g</span> <span class="o">=</span> <span class="n">autograd_one_layer</span><span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">3</span> <span class="nb">print</span><span class="p">(</span><span class="n">W_g</span><span class="p">,</span> <span class="n">b_g</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/wrap_util.py:20,</span> in <span class="ni">unary_to_nary.<locals>.nary_operator.<locals>.nary_f</span><span class="nt">(*args, **kwargs)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">18</span> <span class="k">else</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">19</span> <span class="n">x</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">args</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">argnum</span><span class="p">)</span>
|
||
<span class="ne">---> </span><span class="mi">20</span> <span class="k">return</span> <span class="n">unary_operator</span><span class="p">(</span><span class="n">unary_f</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="o">*</span><span class="n">nary_op_args</span><span class="p">,</span> <span class="o">**</span><span class="n">nary_op_kwargs</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/differential_operators.py:28,</span> in <span class="ni">grad</span><span class="nt">(fun, x)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">21</span> <span class="nd">@unary_to_nary</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">22</span> <span class="k">def</span> <span class="nf">grad</span><span class="p">(</span><span class="n">fun</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">23</span><span class="w"> </span><span class="sd">"""</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">24</span><span class="sd"> Returns a function which computes the gradient of `fun` with respect to</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">25</span><span class="sd"> positional argument number `argnum`. The returned function takes the same</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">26</span><span class="sd"> arguments as `fun`, but returns the gradient instead. The function `fun`</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">27</span><span class="sd"> should be scalar-valued. The gradient has the same type as the argument."""</span>
|
||
<span class="ne">---> </span><span class="mi">28</span> <span class="n">vjp</span><span class="p">,</span> <span class="n">ans</span> <span class="o">=</span> <span class="n">_make_vjp</span><span class="p">(</span><span class="n">fun</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">29</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">vspace</span><span class="p">(</span><span class="n">ans</span><span class="p">)</span><span class="o">.</span><span class="n">size</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">30</span> <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"Grad only applies to real scalar-output functions. "</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">31</span> <span class="s2">"Try jacobian, elementwise_grad or holomorphic_grad."</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/core.py:10,</span> in <span class="ni">make_vjp</span><span class="nt">(fun, x)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">8</span> <span class="k">def</span> <span class="nf">make_vjp</span><span class="p">(</span><span class="n">fun</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">9</span> <span class="n">start_node</span> <span class="o">=</span> <span class="n">VJPNode</span><span class="o">.</span><span class="n">new_root</span><span class="p">()</span>
|
||
<span class="ne">---> </span><span class="mi">10</span> <span class="n">end_value</span><span class="p">,</span> <span class="n">end_node</span> <span class="o">=</span> <span class="n">trace</span><span class="p">(</span><span class="n">start_node</span><span class="p">,</span> <span class="n">fun</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">11</span> <span class="k">if</span> <span class="n">end_node</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">12</span> <span class="k">def</span> <span class="nf">vjp</span><span class="p">(</span><span class="n">g</span><span class="p">):</span> <span class="k">return</span> <span class="n">vspace</span><span class="p">(</span><span class="n">x</span><span class="p">)</span><span class="o">.</span><span class="n">zeros</span><span class="p">()</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/tracer.py:10,</span> in <span class="ni">trace</span><span class="nt">(start_node, fun, x)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">8</span> <span class="k">with</span> <span class="n">trace_stack</span><span class="o">.</span><span class="n">new_trace</span><span class="p">()</span> <span class="k">as</span> <span class="n">t</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">9</span> <span class="n">start_box</span> <span class="o">=</span> <span class="n">new_box</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">t</span><span class="p">,</span> <span class="n">start_node</span><span class="p">)</span>
|
||
<span class="ne">---> </span><span class="mi">10</span> <span class="n">end_box</span> <span class="o">=</span> <span class="n">fun</span><span class="p">(</span><span class="n">start_box</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">11</span> <span class="k">if</span> <span class="n">isbox</span><span class="p">(</span><span class="n">end_box</span><span class="p">)</span> <span class="ow">and</span> <span class="n">end_box</span><span class="o">.</span><span class="n">_trace</span> <span class="o">==</span> <span class="n">start_box</span><span class="o">.</span><span class="n">_trace</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">12</span> <span class="k">return</span> <span class="n">end_box</span><span class="o">.</span><span class="n">_value</span><span class="p">,</span> <span class="n">end_box</span><span class="o">.</span><span class="n">_node</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/wrap_util.py:14,</span> in <span class="ni">unary_to_nary.<locals>.nary_operator.<locals>.nary_f.<locals>.unary_f</span><span class="nt">(x)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">12</span> <span class="n">subargs</span> <span class="o">=</span> <span class="n">subvals</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="p">[(</span><span class="n">argnum</span><span class="p">,</span> <span class="n">x</span><span class="p">)])</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">13</span> <span class="k">else</span><span class="p">:</span>
|
||
<span class="ne">---> </span><span class="mi">14</span> <span class="n">subargs</span> <span class="o">=</span> <span class="n">subvals</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="nb">zip</span><span class="p">(</span><span class="n">argnum</span><span class="p">,</span> <span class="n">x</span><span class="p">))</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">15</span> <span class="k">return</span> <span class="n">fun</span><span class="p">(</span><span class="o">*</span><span class="n">subargs</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/util.py:6,</span> in <span class="ni">subvals</span><span class="nt">(x, ivs)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">4</span> <span class="k">def</span> <span class="nf">subvals</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">ivs</span><span class="p">):</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">5</span> <span class="n">x_</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="ne">----> </span><span class="mi">6</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">ivs</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">7</span> <span class="n">x_</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">v</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">8</span> <span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">x_</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/tracer.py:46,</span> in <span class="ni">primitive.<locals>.f_wrapped</span><span class="nt">(*args, **kwargs)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">44</span> <span class="n">ans</span> <span class="o">=</span> <span class="n">f_wrapped</span><span class="p">(</span><span class="o">*</span><span class="n">argvals</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">45</span> <span class="n">node</span> <span class="o">=</span> <span class="n">node_constructor</span><span class="p">(</span><span class="n">ans</span><span class="p">,</span> <span class="n">f_wrapped</span><span class="p">,</span> <span class="n">argvals</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">,</span> <span class="n">argnums</span><span class="p">,</span> <span class="n">parents</span><span class="p">)</span>
|
||
<span class="ne">---> </span><span class="mi">46</span> <span class="k">return</span> <span class="n">new_box</span><span class="p">(</span><span class="n">ans</span><span class="p">,</span> <span class="n">trace</span><span class="p">,</span> <span class="n">node</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">47</span> <span class="k">else</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">48</span> <span class="k">return</span> <span class="n">f_raw</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/tracer.py:120,</span> in <span class="ni">new_box</span><span class="nt">(value, trace, node)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">118</span> <span class="k">return</span> <span class="n">box_type_mappings</span><span class="p">[</span><span class="nb">type</span><span class="p">(</span><span class="n">value</span><span class="p">)](</span><span class="n">value</span><span class="p">,</span> <span class="n">trace</span><span class="p">,</span> <span class="n">node</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">119</span> <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
|
||
<span class="ne">--> </span><span class="mi">120</span> <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"Can't differentiate w.r.t. type </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">value</span><span class="p">)))</span>
|
||
|
||
<span class="ne">TypeError</span>: Can't differentiate w.r.t. type <class 'ellipsis'>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="tex2jax_ignore mathjax_ignore section" id="exercise-3-gradient-with-one-layer-writing-backpropagation-by-hand">
|
||
<h1>Exercise 3 - Gradient with one layer writing backpropagation by hand<a class="headerlink" href="#exercise-3-gradient-with-one-layer-writing-backpropagation-by-hand" title="Permalink to this headline">¶</a></h1>
|
||
<p>Before you use the gradient you found using autograd, you will have to find the gradient “manually”, to better understand how the backpropagation computation works. To do backpropagation “manually”, you will need to write out expressions for many derivatives along the computation.</p>
|
||
<p>We want to find the gradient of the cost function wrt. the weight and bias. This is quite hard to do directly, so we instead use the chain rule to combine multiple derivatives which are easier to compute.</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{dC}{dW} = \frac{dC}{da}\frac{da}{dz}\frac{dz}{dW}
|
||
\]</div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{dC}{db} = \frac{dC}{da}\frac{da}{dz}\frac{dz}{db}
|
||
\]</div>
|
||
<p><strong>a)</strong> Which intermediary results can be reused between the two expressions?</p>
|
||
<p><strong>b)</strong> What is the derivative of the cost wrt. the final activation? You can use the autograd calculation to make sure you get the correct result. Remember that we compute the mean in mse.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">z</span> <span class="o">=</span> <span class="n">W</span> <span class="o">@</span> <span class="n">x</span> <span class="o">+</span> <span class="n">b</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="n">sigmoid</span><span class="p">(</span><span class="n">z</span><span class="p">)</span>
|
||
|
||
<span class="n">predict</span> <span class="o">=</span> <span class="n">a</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">mse_der</span><span class="p">(</span><span class="n">predict</span><span class="p">,</span> <span class="n">target</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="o">...</span>
|
||
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">mse_der</span><span class="p">(</span><span class="n">predict</span><span class="p">,</span> <span class="n">target</span><span class="p">))</span>
|
||
|
||
<span class="n">cost_autograd</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">mse</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">cost_autograd</span><span class="p">(</span><span class="n">predict</span><span class="p">,</span> <span class="n">target</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>c)</strong> What is the expression for the derivative of the sigmoid activation function? You can use the autograd calculation to make sure you get the correct result.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">sigmoid_der</span><span class="p">(</span><span class="n">z</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="o">...</span>
|
||
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">sigmoid_der</span><span class="p">(</span><span class="n">z</span><span class="p">))</span>
|
||
|
||
<span class="n">sigmoid_autograd</span> <span class="o">=</span> <span class="n">elementwise_grad</span><span class="p">(</span><span class="n">sigmoid</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">sigmoid_autograd</span><span class="p">(</span><span class="n">z</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>d)</strong> Using the two derivatives you just computed, compute this intermetidary gradient you will use later:</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{dC}{dz} = \frac{dC}{da}\frac{da}{dz}
|
||
\]</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">dC_da</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_dz</span> <span class="o">=</span> <span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>e)</strong> What is the derivative of the intermediary z wrt. the weight and bias? What should the shapes be? The one for the weights is a little tricky, it can be easier to play around in the next exercise first. You can also try computing it with autograd to get a hint.</p>
|
||
<p><strong>f)</strong> Now combine the expressions you have worked with so far to compute the gradients! Note that you always need to do a feed forward pass while saving the zs and as before you do backpropagation, as they are used in the derivative expressions</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">dC_da</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_dz</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_dW</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_db</span> <span class="o">=</span> <span class="o">...</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">dC_dW</span><span class="p">,</span> <span class="n">dC_db</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>You should get the same results as with autograd.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">W_g</span><span class="p">,</span> <span class="n">b_g</span> <span class="o">=</span> <span class="n">autograd_one_layer</span><span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">W_g</span><span class="p">,</span> <span class="n">b_g</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="tex2jax_ignore mathjax_ignore section" id="exercise-4-gradient-with-two-layers-writing-backpropagation-by-hand">
|
||
<h1>Exercise 4 - Gradient with two layers writing backpropagation by hand<a class="headerlink" href="#exercise-4-gradient-with-two-layers-writing-backpropagation-by-hand" title="Permalink to this headline">¶</a></h1>
|
||
<p>Now that you have implemented backpropagation for one layer, you have found most of the expressions you will need for more layers. Let’s move up to two layers.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
|
||
<span class="n">target</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">4</span><span class="p">)</span>
|
||
|
||
<span class="n">W1</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
|
||
<span class="n">b1</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span>
|
||
|
||
<span class="n">W2</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
|
||
<span class="n">b2</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">4</span><span class="p">)</span>
|
||
|
||
<span class="n">layers</span> <span class="o">=</span> <span class="p">[(</span><span class="n">W1</span><span class="p">,</span> <span class="n">b1</span><span class="p">),</span> <span class="p">(</span><span class="n">W2</span><span class="p">,</span> <span class="n">b2</span><span class="p">)]</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">z1</span> <span class="o">=</span> <span class="n">W1</span> <span class="o">@</span> <span class="n">x</span> <span class="o">+</span> <span class="n">b1</span>
|
||
<span class="n">a1</span> <span class="o">=</span> <span class="n">sigmoid</span><span class="p">(</span><span class="n">z1</span><span class="p">)</span>
|
||
<span class="n">z2</span> <span class="o">=</span> <span class="n">W2</span> <span class="o">@</span> <span class="n">a1</span> <span class="o">+</span> <span class="n">b2</span>
|
||
<span class="n">a2</span> <span class="o">=</span> <span class="n">sigmoid</span><span class="p">(</span><span class="n">z2</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We begin by computing the gradients of the last layer, as the gradients must be propagated backwards from the end.</p>
|
||
<p><strong>a)</strong> Compute the gradients of the last layer, just like you did the single layer in the previous exercise.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">dC_da2</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_dz2</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_dW2</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_db2</span> <span class="o">=</span> <span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>To find the derivative of the cost wrt. the activation of the first layer, we need a new expression, the one furthest to the right in the following.</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{dC}{da_1} = \frac{dC}{dz_2}\frac{dz_2}{da_1}
|
||
\]</div>
|
||
<p><strong>b)</strong> What is the derivative of the second layer intermetiate wrt. the first layer activation? (First recall how you compute <span class="math notranslate nohighlight">\(z_2\)</span>)</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{dz_2}{da_1}
|
||
\]</div>
|
||
<p><strong>c)</strong> Use this expression, together with expressions which are equivelent to ones for the last layer to compute all the derivatives of the first layer.</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{dC}{dW_1} = \frac{dC}{da_1}\frac{da_1}{dz_1}\frac{dz_1}{dW_1}
|
||
\]</div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{dC}{db_1} = \frac{dC}{da_1}\frac{da_1}{dz_1}\frac{dz_1}{db_1}
|
||
\]</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">dC_da1</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_dz1</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_dW1</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_db1</span> <span class="o">=</span> <span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="nb">print</span><span class="p">(</span><span class="n">dC_dW1</span><span class="p">,</span> <span class="n">dC_db1</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">dC_dW2</span><span class="p">,</span> <span class="n">dC_db2</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>d)</strong> Make sure you got the same gradient as the following code which uses autograd to do backpropagation.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">feed_forward_two_layers</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
|
||
<span class="n">W1</span><span class="p">,</span> <span class="n">b1</span> <span class="o">=</span> <span class="n">layers</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||
<span class="n">z1</span> <span class="o">=</span> <span class="n">W1</span> <span class="o">@</span> <span class="n">x</span> <span class="o">+</span> <span class="n">b1</span>
|
||
<span class="n">a1</span> <span class="o">=</span> <span class="n">sigmoid</span><span class="p">(</span><span class="n">z1</span><span class="p">)</span>
|
||
|
||
<span class="n">W2</span><span class="p">,</span> <span class="n">b2</span> <span class="o">=</span> <span class="n">layers</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
||
<span class="n">z2</span> <span class="o">=</span> <span class="n">W2</span> <span class="o">@</span> <span class="n">a1</span> <span class="o">+</span> <span class="n">b2</span>
|
||
<span class="n">a2</span> <span class="o">=</span> <span class="n">sigmoid</span><span class="p">(</span><span class="n">z2</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">a2</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">cost_two_layers</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">target</span><span class="p">):</span>
|
||
<span class="n">predict</span> <span class="o">=</span> <span class="n">feed_forward_two_layers</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">mse</span><span class="p">(</span><span class="n">predict</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
|
||
|
||
|
||
<span class="n">grad_two_layers</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">cost_two_layers</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||
<span class="n">grad_two_layers</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>e)</strong> How would you use the gradient from this layer to compute the gradient of an even earlier layer? Would the expressions be any different?</p>
|
||
</div>
|
||
<div class="tex2jax_ignore mathjax_ignore section" id="exercise-5-gradient-with-any-number-of-layers-writing-backpropagation-by-hand">
|
||
<h1>Exercise 5 - Gradient with any number of layers writing backpropagation by hand<a class="headerlink" href="#exercise-5-gradient-with-any-number-of-layers-writing-backpropagation-by-hand" title="Permalink to this headline">¶</a></h1>
|
||
<p>Well done on getting this far! Now it’s time to compute the gradient with any number of layers.</p>
|
||
<p>First, some code from the general neural network code from last week. Note that we are still sending in one input vector at a time. We will change it to use batched inputs later.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">create_layers</span><span class="p">(</span><span class="n">network_input_size</span><span class="p">,</span> <span class="n">layer_output_sizes</span><span class="p">):</span>
|
||
<span class="n">layers</span> <span class="o">=</span> <span class="p">[]</span>
|
||
|
||
<span class="n">i_size</span> <span class="o">=</span> <span class="n">network_input_size</span>
|
||
<span class="k">for</span> <span class="n">layer_output_size</span> <span class="ow">in</span> <span class="n">layer_output_sizes</span><span class="p">:</span>
|
||
<span class="n">W</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">layer_output_size</span><span class="p">,</span> <span class="n">i_size</span><span class="p">)</span>
|
||
<span class="n">b</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">layer_output_size</span><span class="p">)</span>
|
||
<span class="n">layers</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">))</span>
|
||
|
||
<span class="n">i_size</span> <span class="o">=</span> <span class="n">layer_output_size</span>
|
||
<span class="k">return</span> <span class="n">layers</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">feed_forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">):</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="nb">input</span>
|
||
<span class="k">for</span> <span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">),</span> <span class="n">activation_func</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">):</span>
|
||
<span class="n">z</span> <span class="o">=</span> <span class="n">W</span> <span class="o">@</span> <span class="n">a</span> <span class="o">+</span> <span class="n">b</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="n">activation_func</span><span class="p">(</span><span class="n">z</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">a</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">cost</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="nb">input</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">,</span> <span class="n">target</span><span class="p">):</span>
|
||
<span class="n">predict</span> <span class="o">=</span> <span class="n">feed_forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">mse</span><span class="p">(</span><span class="n">predict</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>You might have already have noticed a very important detail in backpropagation: You need the values from the forward pass to compute all the gradients! The feed forward method above is great for efficiency and for using autograd, as it only cares about computing the final output, but now we need to also save the results along the way.</p>
|
||
<p>Here is a function which does that for you.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">feed_forward_saver</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">):</span>
|
||
<span class="n">layer_inputs</span> <span class="o">=</span> <span class="p">[]</span>
|
||
<span class="n">zs</span> <span class="o">=</span> <span class="p">[]</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="nb">input</span>
|
||
<span class="k">for</span> <span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">),</span> <span class="n">activation_func</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">):</span>
|
||
<span class="n">layer_inputs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">a</span><span class="p">)</span>
|
||
<span class="n">z</span> <span class="o">=</span> <span class="n">W</span> <span class="o">@</span> <span class="n">a</span> <span class="o">+</span> <span class="n">b</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="n">activation_func</span><span class="p">(</span><span class="n">z</span><span class="p">)</span>
|
||
|
||
<span class="n">zs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">z</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">layer_inputs</span><span class="p">,</span> <span class="n">zs</span><span class="p">,</span> <span class="n">a</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>a)</strong> Now, complete the backpropagation function so that it returns the gradient of the cost function wrt. all the weigths and biases. Use the autograd calculation below to make sure you get the correct answer.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">backpropagation</span><span class="p">(</span>
|
||
<span class="nb">input</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">activation_ders</span><span class="p">,</span> <span class="n">cost_der</span><span class="o">=</span><span class="n">mse_der</span>
|
||
<span class="p">):</span>
|
||
<span class="n">layer_inputs</span><span class="p">,</span> <span class="n">zs</span><span class="p">,</span> <span class="n">predict</span> <span class="o">=</span> <span class="n">feed_forward_saver</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">)</span>
|
||
|
||
<span class="n">layer_grads</span> <span class="o">=</span> <span class="p">[()</span> <span class="k">for</span> <span class="n">layer</span> <span class="ow">in</span> <span class="n">layers</span><span class="p">]</span>
|
||
|
||
<span class="c1"># We loop over the layers, from the last to the first</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">reversed</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">layers</span><span class="p">))):</span>
|
||
<span class="n">layer_input</span><span class="p">,</span> <span class="n">z</span><span class="p">,</span> <span class="n">activation_der</span> <span class="o">=</span> <span class="n">layer_inputs</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">zs</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">activation_ders</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||
|
||
<span class="k">if</span> <span class="n">i</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">layers</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">:</span>
|
||
<span class="c1"># For last layer we use cost derivative as dC_da(L) can be computed directly</span>
|
||
<span class="n">dC_da</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="c1"># For other layers we build on previous z derivative, as dC_da(i) = dC_dz(i+1) * dz(i+1)_da(i)</span>
|
||
<span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">)</span> <span class="o">=</span> <span class="n">layers</span><span class="p">[</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span>
|
||
<span class="n">dC_da</span> <span class="o">=</span> <span class="o">...</span>
|
||
|
||
<span class="n">dC_dz</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_dW</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_db</span> <span class="o">=</span> <span class="o">...</span>
|
||
|
||
<span class="n">layer_grads</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">dC_dW</span><span class="p">,</span> <span class="n">dC_db</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">layer_grads</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">network_input_size</span> <span class="o">=</span> <span class="mi">2</span>
|
||
<span class="n">layer_output_sizes</span> <span class="o">=</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]</span>
|
||
<span class="n">activation_funcs</span> <span class="o">=</span> <span class="p">[</span><span class="n">sigmoid</span><span class="p">,</span> <span class="n">ReLU</span><span class="p">]</span>
|
||
<span class="n">activation_ders</span> <span class="o">=</span> <span class="p">[</span><span class="n">sigmoid_der</span><span class="p">,</span> <span class="n">ReLU_der</span><span class="p">]</span>
|
||
|
||
<span class="n">layers</span> <span class="o">=</span> <span class="n">create_layers</span><span class="p">(</span><span class="n">network_input_size</span><span class="p">,</span> <span class="n">layer_output_sizes</span><span class="p">)</span>
|
||
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">network_input_size</span><span class="p">)</span>
|
||
<span class="n">target</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">4</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">layer_grads</span> <span class="o">=</span> <span class="n">backpropagation</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">activation_ders</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">layer_grads</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">cost_grad</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">cost</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||
<span class="n">cost_grad</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="p">[</span><span class="n">sigmoid</span><span class="p">,</span> <span class="n">ReLU</span><span class="p">],</span> <span class="n">target</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="tex2jax_ignore mathjax_ignore section" id="exercise-6-batched-inputs">
|
||
<h1>Exercise 6 - Batched inputs<a class="headerlink" href="#exercise-6-batched-inputs" title="Permalink to this headline">¶</a></h1>
|
||
<p>Make new versions of all the functions in exercise 5 which now take batched inputs instead. See last weeks exercise 5 for details on how to batch inputs to neural networks. You will also need to update the backpropogation function.</p>
|
||
</div>
|
||
<div class="tex2jax_ignore mathjax_ignore section" id="exercise-7-training">
|
||
<h1>Exercise 7 - Training<a class="headerlink" href="#exercise-7-training" title="Permalink to this headline">¶</a></h1>
|
||
<p><strong>a)</strong> Complete exercise 6 and 7 from last week, but use your own backpropogation implementation to compute the gradient.</p>
|
||
<p><strong>b)</strong> Use stochastic gradient descent with momentum when you train your network.</p>
|
||
</div>
|
||
<div class="tex2jax_ignore mathjax_ignore section" id="exercise-8-optional-object-orientation">
|
||
<h1>Exercise 8 (Optional) - Object orientation<a class="headerlink" href="#exercise-8-optional-object-orientation" title="Permalink to this headline">¶</a></h1>
|
||
<p>Passing in the layers, activations functions, activation derivatives and cost derivatives into the functions each time leads to code which is easy to understand in isoloation, but messier when used in a larger context with data splitting, data scaling, gradient methods and so forth. Creating an object which stores these values can lead to code which is much easier to use.</p>
|
||
<p><strong>a)</strong> Write a neural network class. You are free to implement it how you see fit, though we strongly recommend to not save any input or output values as class attributes, nor let the neural network class handle gradient methods internally. Gradient methods should be handled outside, by performing general operations on the layer_grads list using functions or classes separate to the neural network.</p>
|
||
<p>We provide here a skeleton structure which should get you started.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">NeuralNetwork</span><span class="p">:</span>
|
||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
|
||
<span class="bp">self</span><span class="p">,</span>
|
||
<span class="n">network_input_size</span><span class="p">,</span>
|
||
<span class="n">layer_output_sizes</span><span class="p">,</span>
|
||
<span class="n">activation_funcs</span><span class="p">,</span>
|
||
<span class="n">activation_ders</span><span class="p">,</span>
|
||
<span class="n">cost_fun</span><span class="p">,</span>
|
||
<span class="n">cost_der</span><span class="p">,</span>
|
||
<span class="p">):</span>
|
||
<span class="k">pass</span>
|
||
|
||
<span class="k">def</span> <span class="nf">predict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inputs</span><span class="p">):</span>
|
||
<span class="c1"># Simple feed forward pass</span>
|
||
<span class="k">pass</span>
|
||
|
||
<span class="k">def</span> <span class="nf">cost</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">targets</span><span class="p">):</span>
|
||
<span class="k">pass</span>
|
||
|
||
<span class="k">def</span> <span class="nf">_feed_forward_saver</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inputs</span><span class="p">):</span>
|
||
<span class="k">pass</span>
|
||
|
||
<span class="k">def</span> <span class="nf">compute_gradient</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">targets</span><span class="p">):</span>
|
||
<span class="k">pass</span>
|
||
|
||
<span class="k">def</span> <span class="nf">update_weights</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">layer_grads</span><span class="p">):</span>
|
||
<span class="k">pass</span>
|
||
|
||
<span class="c1"># These last two methods are not needed in the project, but they can be nice to have! The first one has a layers parameter so that you can use autograd on it</span>
|
||
<span class="k">def</span> <span class="nf">autograd_compliant_predict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">inputs</span><span class="p">):</span>
|
||
<span class="k">pass</span>
|
||
|
||
<span class="k">def</span> <span class="nf">autograd_gradient</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">targets</span><span class="p">):</span>
|
||
<span class="k">pass</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
kernelName: "python3",
|
||
path: "./."
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</div>
|
||
|
||
|
||
<!-- Previous / next buttons -->
|
||
<div class='prev-next-area'>
|
||
<a class='left-prev' id="prev-link" href="week43.html" title="previous page">
|
||
<i class="fas fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title">Week 43: Deep Learning: Constructing a Neural Network code and solving differential equations</p>
|
||
</div>
|
||
</a>
|
||
<a class='right-next' id="next-link" href="week44.html" title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title">Week 44, Convolutional Neural Networks (CNN)</p>
|
||
</div>
|
||
<i class="fas fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
<footer class="footer">
|
||
<p>
|
||
|
||
By Morten Hjorth-Jensen<br/>
|
||
|
||
© Copyright 2021.<br/>
|
||
</p>
|
||
</footer>
|
||
</main>
|
||
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<script src="_static/js/index.be7d3bbb2ef33a8344ce.js"></script>
|
||
|
||
</body>
|
||
</html> |