Morten Hjorth-Jensen
1122 lines
85 KiB
HTML
1122 lines
85 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
|
||
<html lang="en" data-content_root="./" >
|
||
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<title>Exercises week 43 — Applied Data Analysis and Machine Learning</title>
|
||
|
||
|
||
|
||
<script data-cfasync="false">
|
||
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
||
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
||
</script>
|
||
<!--
|
||
this give us a css class that will be invisible only if js is disabled
|
||
-->
|
||
<noscript>
|
||
<style>
|
||
.pst-js-only { display: none !important; }
|
||
|
||
</style>
|
||
</noscript>
|
||
|
||
<!-- Loaded before other Sphinx assets -->
|
||
<link href="_static/styles/theme.css?digest=26a4bc78f4c0ddb94549" rel="stylesheet" />
|
||
<link href="_static/styles/pydata-sphinx-theme.css?digest=26a4bc78f4c0ddb94549" rel="stylesheet" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=fa44fd50" />
|
||
<link rel="stylesheet" type="text/css" href="_static/styles/sphinx-book-theme.css?v=a3416100" />
|
||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css?v=13237357" />
|
||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css?v=76b2166b" />
|
||
<link rel="stylesheet" type="text/css" href="_static/mystnb.4510f1fc1dee50b3e5859aac5469c37c29e427902b24a333a5f9fcb2f0b3ac41.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css?v=4fa983c6" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-design.min.css?v=95c83b7e" />
|
||
|
||
<!-- So that users can add custom icons -->
|
||
<script src="_static/scripts/fontawesome.js?digest=26a4bc78f4c0ddb94549"></script>
|
||
<!-- Pre-loaded scripts that we'll load fully later -->
|
||
<link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=26a4bc78f4c0ddb94549" />
|
||
<link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=26a4bc78f4c0ddb94549" />
|
||
|
||
<script src="_static/documentation_options.js?v=9eb32ce0"></script>
|
||
<script src="_static/doctools.js?v=9a2dae69"></script>
|
||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
<script src="_static/clipboard.min.js?v=a7894cd8"></script>
|
||
<script src="_static/copybutton.js?v=f281be69"></script>
|
||
<script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="_static/togglebutton.js?v=4a39c7ea"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script src="_static/design-tabs.js?v=f930bc37"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script async="async" src="_static/sphinx-thebe.js?v=c100c467"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
||
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
<script>DOCUMENTATION_OPTIONS.pagename = 'exercisesweek43';</script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="Week 44, Convolutional Neural Networks (CNN)" href="week44.html" />
|
||
<link rel="prev" title="Week 43: Deep Learning: Constructing a Neural Network code and solving differential equations" href="week43.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||
<meta name="docsearch:language" content="en"/>
|
||
<meta name="docsearch:version" content="" />
|
||
</head>
|
||
|
||
|
||
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
||
|
||
|
||
|
||
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
||
|
||
<div id="pst-scroll-pixel-helper"></div>
|
||
|
||
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
||
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
||
|
||
|
||
<dialog id="pst-search-dialog">
|
||
|
||
<form class="bd-search d-flex align-items-center"
|
||
action="search.html"
|
||
method="get">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<input type="search"
|
||
class="form-control"
|
||
name="q"
|
||
placeholder="Search this book..."
|
||
aria-label="Search this book..."
|
||
autocomplete="off"
|
||
autocorrect="off"
|
||
autocapitalize="off"
|
||
spellcheck="false"/>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
||
</form>
|
||
</dialog>
|
||
|
||
<div class="pst-async-banner-revealer d-none">
|
||
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
||
</div>
|
||
|
||
|
||
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
||
</header>
|
||
|
||
|
||
<div class="bd-container">
|
||
<div class="bd-container__inner bd-page-width">
|
||
|
||
|
||
|
||
<dialog id="pst-primary-sidebar-modal"></dialog>
|
||
<div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
|
||
|
||
|
||
|
||
<div class="sidebar-header-items sidebar-primary__section">
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
<div class="sidebar-primary-items__start sidebar-primary__section">
|
||
<div class="sidebar-primary-item">
|
||
|
||
|
||
|
||
|
||
|
||
<a class="navbar-brand logo" href="intro.html">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<img src="_static/logo.png" class="logo__image only-light" alt="Applied Data Analysis and Machine Learning - Home"/>
|
||
<img src="_static/logo.png" class="logo__image only-dark pst-js-only" alt="Applied Data Analysis and Machine Learning - Home"/>
|
||
|
||
|
||
</a></div>
|
||
<div class="sidebar-primary-item">
|
||
|
||
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<span class="search-button__default-text">Search</span>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||
</button></div>
|
||
<div class="sidebar-primary-item"><nav class="bd-links bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item navbar-nav active">
|
||
|
||
<ul class="nav bd-sidenav bd-sidenav__home-link">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="intro.html">
|
||
Applied Data Analysis and Machine Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">About the course</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="schedule.html">Teaching schedule with links to material</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="teachers.html">Teachers and Grading</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="textbooks.html">Textbooks</a></li>
|
||
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Review of Statistics with Resampling Techniques and Linear Algebra</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="statistics.html">1. Elements of Probability Theory and Statistical Data Analysis</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="linalg.html">2. Linear Algebra, Handling of Arrays and more Python Features</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">From Regression to Support Vector Machines</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter1.html">3. Linear Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter2.html">4. Ridge and Lasso Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter3.html">5. Resampling Methods</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter4.html">6. Logistic Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapteroptimization.html">7. Optimization, the central part of any Machine Learning algortithm</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter5.html">8. Support Vector Machines, overarching aims</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Decision Trees, Ensemble Methods and Boosting</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter6.html">9. Decision trees, overarching aims</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter7.html">10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Dimensionality Reduction</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter8.html">11. Basic ideas of the Principal Component Analysis (PCA)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="clustering.html">12. Clustering and Unsupervised Learning</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Deep Learning Methods</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter9.html">13. Neural networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter10.html">14. Building a Feed Forward Neural Network</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter11.html">15. Solving Differential Equations with Deep Learning</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter12.html">16. Convolutional Neural Networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter13.html">17. Recurrent neural networks: Overarching view</a></li>
|
||
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Weekly material, notes and exercises</span></p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek34.html">Exercises week 34</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week34.html">Week 34: Introduction to the course, Logistics and Practicalities</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek35.html">Exercises week 35</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week35.html">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Statistical interpretations</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 38</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week38.html">Week 38: Logistic Regression and Optimization</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek39.html">Exercises week 39</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week39.html">Week 39: Optimization and Gradient Methods</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week40.html">Week 40: Gradient descent methods (continued) and start Neural networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek41.html">Exercises week 41</a></li>
|
||
|
||
|
||
<li class="toctree-l1"><a class="reference internal" href="week41.html">Week 41 Neural networks and constructing a neural network code</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek42.html">Exercises week 42</a></li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="toctree-l1"><a class="reference internal" href="week42.html">Week 42 Constructing a Neural Network code with examples</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="additionweek42.html">Exercises Week 42: Logistic Regression and Optimization, reminders from week 38 and week 40</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week43.html">Week 43: Deep Learning: Constructing a Neural Network code and solving differential equations</a></li>
|
||
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Exercises week 43</a></li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="toctree-l1"><a class="reference internal" href="week44.html">Week 44, Convolutional Neural Networks (CNN)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week45.html">Week 45, Convolutional Neural Networks (CCNs) and Recurrent Neural Networks (RNNs)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week46.html">Week 46: Decision Trees, Ensemble methods and Random Forests</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="project1.html">Project 1 on Machine Learning, deadline October 7 (midnight), 2024</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="project2.html">Project 2 on Machine Learning, deadline November 4 (Midnight)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="project3.html">Project 3 on Machine Learning, deadline December 9 (midnight), 2024</a></li>
|
||
|
||
</ul>
|
||
|
||
</div>
|
||
</nav></div>
|
||
</div>
|
||
|
||
|
||
<div class="sidebar-primary-items__end sidebar-primary__section">
|
||
</div>
|
||
|
||
<div id="rtd-footer-container"></div>
|
||
|
||
|
||
</div>
|
||
|
||
<main id="main-content" class="bd-main" role="main">
|
||
|
||
|
||
|
||
<div class="sbt-scroll-pixel-helper"></div>
|
||
|
||
<div class="bd-content">
|
||
<div class="bd-article-container">
|
||
|
||
<div class="bd-header-article d-print-none">
|
||
<div class="header-article-items header-article__inner">
|
||
|
||
<div class="header-article-items__start">
|
||
|
||
<div class="header-article-item"><button class="sidebar-toggle primary-toggle btn btn-sm" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-bars"></span>
|
||
</button></div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="header-article-items__end">
|
||
|
||
<div class="header-article-item">
|
||
|
||
<div class="article-header-buttons">
|
||
|
||
|
||
|
||
|
||
|
||
<div class="dropdown dropdown-launch-buttons">
|
||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Launch interactive content">
|
||
<i class="fas fa-rocket"></i>
|
||
</button>
|
||
<ul class="dropdown-menu">
|
||
|
||
|
||
|
||
<li><a href="https://mybinder.org/v2/git/https%3A//compphysics.github.io/MachineLearning/doc/LectureNotes/_build/html/index.html/master?urlpath=tree/exercisesweek43.ipynb" target="_blank"
|
||
class="btn btn-sm dropdown-item"
|
||
title="Launch on Binder"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
|
||
<img alt="Binder logo" src="_static/images/logo_binder.svg">
|
||
</span>
|
||
<span class="btn__text-container">Binder</span>
|
||
</a>
|
||
</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="dropdown dropdown-download-buttons">
|
||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
|
||
<i class="fas fa-download"></i>
|
||
</button>
|
||
<ul class="dropdown-menu">
|
||
|
||
|
||
|
||
<li><a href="_sources/exercisesweek43.ipynb" target="_blank"
|
||
class="btn btn-sm btn-download-source-button dropdown-item"
|
||
title="Download source file"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file"></i>
|
||
</span>
|
||
<span class="btn__text-container">.ipynb</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li>
|
||
<button onclick="window.print()"
|
||
class="btn btn-sm btn-download-pdf-button dropdown-item"
|
||
title="Print to PDF"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file-pdf"></i>
|
||
</span>
|
||
<span class="btn__text-container">.pdf</span>
|
||
</button>
|
||
</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<button onclick="toggleFullScreen()"
|
||
class="btn btn-sm btn-fullscreen-button"
|
||
title="Fullscreen mode"
|
||
data-bs-placement="bottom" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-expand"></i>
|
||
</span>
|
||
|
||
</button>
|
||
|
||
|
||
|
||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
|
||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
|
||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
|
||
</button>
|
||
|
||
|
||
<button class="btn btn-sm pst-navbar-icon search-button search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass fa-lg"></i>
|
||
</button>
|
||
<button class="sidebar-toggle secondary-toggle btn btn-sm" title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-list"></span>
|
||
</button>
|
||
</div></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>Exercises week 43</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#">Exercises week 43</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#overarching-aims-of-the-exercises-this-week">Overarching aims of the exercises this week</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-1-understand-the-feed-forward-pass">Exercise 1 - Understand the feed forward pass</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-2-gradient-with-one-layer-using-autograd">Exercise 2 - Gradient with one layer using autograd</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-3-gradient-with-one-layer-writing-backpropagation-by-hand">Exercise 3 - Gradient with one layer writing backpropagation by hand</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-4-gradient-with-two-layers-writing-backpropagation-by-hand">Exercise 4 - Gradient with two layers writing backpropagation by hand</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-5-gradient-with-any-number-of-layers-writing-backpropagation-by-hand">Exercise 5 - Gradient with any number of layers writing backpropagation by hand</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-6-batched-inputs">Exercise 6 - Batched inputs</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-7-training">Exercise 7 - Training</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-8-optional-object-orientation">Exercise 8 (Optional) - Object orientation</a></li>
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="searchbox"></div>
|
||
<article class="bd-article">
|
||
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercises-week-43">
|
||
<h1>Exercises week 43<a class="headerlink" href="#exercises-week-43" title="Link to this heading">#</a></h1>
|
||
<p><strong>October 18-25, 2024</strong></p>
|
||
<p>Date: <strong>Deadline is Friday October 25 at midnight</strong></p>
|
||
</section>
|
||
<section class="tex2jax_ignore mathjax_ignore" id="overarching-aims-of-the-exercises-this-week">
|
||
<h1>Overarching aims of the exercises this week<a class="headerlink" href="#overarching-aims-of-the-exercises-this-week" title="Link to this heading">#</a></h1>
|
||
<p>The aim of the exercises this week is to train the neural network you implemented last week.</p>
|
||
<p>To train neural networks, we use gradient descent, since there is no analytical expression for the optimal parameters. This means you will need to compute the gradient of the cost function wrt. the network parameters. And then you will need to implement some gradient method.</p>
|
||
<p>You will begin by computing gradients for a network with one layer, then two layers, then any number of layers. Keeping track of the shapes and doing things step by step will be very important this week.</p>
|
||
<p>We recommend that you do the exercises this week by editing and running this notebook file, as it includes some checks along the way that you have implemented the neural network correctly, and running small parts of the code at a time will be important for understanding the methods. If you have trouble running a notebook, you can run this notebook in google colab instead(<a class="reference external" href="https://colab.research.google.com/drive/1FfvbN0XlhV-lATRPyGRTtTBnJr3zNuHL#offline=true&amp;sandboxMode=true">https://colab.research.google.com/drive/1FfvbN0XlhV-lATRPyGRTtTBnJr3zNuHL#offline=true&sandboxMode=true</a>), though we recommend that you set up VSCode and your python environment to run code like this locally.</p>
|
||
<p>First, some setup code that you will need.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">autograd.numpy</span> <span class="k">as</span> <span class="nn">np</span> <span class="c1"># We need to use this numpy wrapper to make automatic differentiation work later</span>
|
||
<span class="kn">from</span> <span class="nn">autograd</span> <span class="kn">import</span> <span class="n">grad</span><span class="p">,</span> <span class="n">elementwise_grad</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn</span> <span class="kn">import</span> <span class="n">datasets</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">accuracy_score</span>
|
||
|
||
|
||
<span class="c1"># Defining some activation functions</span>
|
||
<span class="k">def</span> <span class="nf">ReLU</span><span class="p">(</span><span class="n">z</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">where</span><span class="p">(</span><span class="n">z</span> <span class="o">></span> <span class="mi">0</span><span class="p">,</span> <span class="n">z</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||
|
||
|
||
<span class="c1"># Derivative of the ReLU function</span>
|
||
<span class="k">def</span> <span class="nf">ReLU_der</span><span class="p">(</span><span class="n">z</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">where</span><span class="p">(</span><span class="n">z</span> <span class="o">></span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">sigmoid</span><span class="p">(</span><span class="n">z</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="mi">1</span> <span class="o">/</span> <span class="p">(</span><span class="mi">1</span> <span class="o">+</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="n">z</span><span class="p">))</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">mse</span><span class="p">(</span><span class="n">predict</span><span class="p">,</span> <span class="n">target</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">((</span><span class="n">predict</span> <span class="o">-</span> <span class="n">target</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercise-1-understand-the-feed-forward-pass">
|
||
<h1>Exercise 1 - Understand the feed forward pass<a class="headerlink" href="#exercise-1-understand-the-feed-forward-pass" title="Link to this heading">#</a></h1>
|
||
<p><strong>a)</strong> Complete last weeks’ mandatory exercises if you haven’t already.</p>
|
||
</section>
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercise-2-gradient-with-one-layer-using-autograd">
|
||
<h1>Exercise 2 - Gradient with one layer using autograd<a class="headerlink" href="#exercise-2-gradient-with-one-layer-using-autograd" title="Link to this heading">#</a></h1>
|
||
<p>For the first few exercises, we will not use batched inputs. Only a single input vector is passed through the layer at a time.</p>
|
||
<p>In this exercise you will compute the gradient of a single layer. You only need to change the code in the cells right below an exercise, the rest works out of the box. Feel free to make changes and see how stuff works though!</p>
|
||
<p><strong>a)</strong> If the weights and bias of a layer has shapes (10, 4) and (10), what will the shapes of the gradients of the cost function wrt. these weights and this bias be?</p>
|
||
<p><strong>b)</strong> Complete the feed_forward_one_layer function. It should use the sigmoid activation function. Also define the weigth and bias with the correct shapes.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">feed_forward_one_layer</span><span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
|
||
<span class="n">z</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="k">return</span> <span class="n">a</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">cost_one_layer</span><span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">target</span><span class="p">):</span>
|
||
<span class="n">predict</span> <span class="o">=</span> <span class="n">feed_forward_one_layer</span><span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">mse</span><span class="p">(</span><span class="n">predict</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
|
||
|
||
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
|
||
<span class="n">target</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span>
|
||
|
||
<span class="n">W</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">b</span> <span class="o">=</span> <span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>c)</strong> Compute the gradient of the cost function wrt. the weigth and bias by running the cell below. You will not need to change anything, just make sure it runs by defining things correctly in the cell above. This code uses the autograd package which uses backprogagation to compute the gradient!</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">autograd_one_layer</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">cost_one_layer</span><span class="p">,</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span>
|
||
<span class="n">W_g</span><span class="p">,</span> <span class="n">b_g</span> <span class="o">=</span> <span class="n">autograd_one_layer</span><span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">W_g</span><span class="p">,</span> <span class="n">b_g</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output traceback highlight-ipythontb notranslate"><div class="highlight"><pre><span></span><span class="gt">---------------------------------------------------------------------------</span>
|
||
<span class="ne">KeyError</span><span class="g g-Whitespace"> </span>Traceback (most recent call last)
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/tracer.py:118,</span> in <span class="ni">new_box</span><span class="nt">(value, trace, node)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">117</span> <span class="k">try</span><span class="p">:</span>
|
||
<span class="ne">--> </span><span class="mi">118</span> <span class="k">return</span> <span class="n">box_type_mappings</span><span class="p">[</span><span class="nb">type</span><span class="p">(</span><span class="n">value</span><span class="p">)](</span><span class="n">value</span><span class="p">,</span> <span class="n">trace</span><span class="p">,</span> <span class="n">node</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">119</span> <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
|
||
|
||
<span class="ne">KeyError</span>: <class 'ellipsis'>
|
||
|
||
<span class="n">During</span> <span class="n">handling</span> <span class="n">of</span> <span class="n">the</span> <span class="n">above</span> <span class="n">exception</span><span class="p">,</span> <span class="n">another</span> <span class="n">exception</span> <span class="n">occurred</span><span class="p">:</span>
|
||
|
||
<span class="ne">TypeError</span><span class="g g-Whitespace"> </span>Traceback (most recent call last)
|
||
<span class="n">Cell</span> <span class="n">In</span><span class="p">[</span><span class="mi">3</span><span class="p">],</span> <span class="n">line</span> <span class="mi">2</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1</span> <span class="n">autograd_one_layer</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">cost_one_layer</span><span class="p">,</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span>
|
||
<span class="ne">----> </span><span class="mi">2</span> <span class="n">W_g</span><span class="p">,</span> <span class="n">b_g</span> <span class="o">=</span> <span class="n">autograd_one_layer</span><span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">3</span> <span class="nb">print</span><span class="p">(</span><span class="n">W_g</span><span class="p">,</span> <span class="n">b_g</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/wrap_util.py:20,</span> in <span class="ni">unary_to_nary.<locals>.nary_operator.<locals>.nary_f</span><span class="nt">(*args, **kwargs)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">18</span> <span class="k">else</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">19</span> <span class="n">x</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">args</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">argnum</span><span class="p">)</span>
|
||
<span class="ne">---> </span><span class="mi">20</span> <span class="k">return</span> <span class="n">unary_operator</span><span class="p">(</span><span class="n">unary_f</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="o">*</span><span class="n">nary_op_args</span><span class="p">,</span> <span class="o">**</span><span class="n">nary_op_kwargs</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/differential_operators.py:28,</span> in <span class="ni">grad</span><span class="nt">(fun, x)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">21</span> <span class="nd">@unary_to_nary</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">22</span> <span class="k">def</span> <span class="nf">grad</span><span class="p">(</span><span class="n">fun</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">23</span><span class="w"> </span><span class="sd">"""</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">24</span><span class="sd"> Returns a function which computes the gradient of `fun` with respect to</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">25</span><span class="sd"> positional argument number `argnum`. The returned function takes the same</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">26</span><span class="sd"> arguments as `fun`, but returns the gradient instead. The function `fun`</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">27</span><span class="sd"> should be scalar-valued. The gradient has the same type as the argument."""</span>
|
||
<span class="ne">---> </span><span class="mi">28</span> <span class="n">vjp</span><span class="p">,</span> <span class="n">ans</span> <span class="o">=</span> <span class="n">_make_vjp</span><span class="p">(</span><span class="n">fun</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">29</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">vspace</span><span class="p">(</span><span class="n">ans</span><span class="p">)</span><span class="o">.</span><span class="n">size</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">30</span> <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"Grad only applies to real scalar-output functions. "</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">31</span> <span class="s2">"Try jacobian, elementwise_grad or holomorphic_grad."</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/core.py:10,</span> in <span class="ni">make_vjp</span><span class="nt">(fun, x)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">8</span> <span class="k">def</span> <span class="nf">make_vjp</span><span class="p">(</span><span class="n">fun</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">9</span> <span class="n">start_node</span> <span class="o">=</span> <span class="n">VJPNode</span><span class="o">.</span><span class="n">new_root</span><span class="p">()</span>
|
||
<span class="ne">---> </span><span class="mi">10</span> <span class="n">end_value</span><span class="p">,</span> <span class="n">end_node</span> <span class="o">=</span> <span class="n">trace</span><span class="p">(</span><span class="n">start_node</span><span class="p">,</span> <span class="n">fun</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">11</span> <span class="k">if</span> <span class="n">end_node</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">12</span> <span class="k">def</span> <span class="nf">vjp</span><span class="p">(</span><span class="n">g</span><span class="p">):</span> <span class="k">return</span> <span class="n">vspace</span><span class="p">(</span><span class="n">x</span><span class="p">)</span><span class="o">.</span><span class="n">zeros</span><span class="p">()</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/tracer.py:10,</span> in <span class="ni">trace</span><span class="nt">(start_node, fun, x)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">8</span> <span class="k">with</span> <span class="n">trace_stack</span><span class="o">.</span><span class="n">new_trace</span><span class="p">()</span> <span class="k">as</span> <span class="n">t</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">9</span> <span class="n">start_box</span> <span class="o">=</span> <span class="n">new_box</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">t</span><span class="p">,</span> <span class="n">start_node</span><span class="p">)</span>
|
||
<span class="ne">---> </span><span class="mi">10</span> <span class="n">end_box</span> <span class="o">=</span> <span class="n">fun</span><span class="p">(</span><span class="n">start_box</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">11</span> <span class="k">if</span> <span class="n">isbox</span><span class="p">(</span><span class="n">end_box</span><span class="p">)</span> <span class="ow">and</span> <span class="n">end_box</span><span class="o">.</span><span class="n">_trace</span> <span class="o">==</span> <span class="n">start_box</span><span class="o">.</span><span class="n">_trace</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">12</span> <span class="k">return</span> <span class="n">end_box</span><span class="o">.</span><span class="n">_value</span><span class="p">,</span> <span class="n">end_box</span><span class="o">.</span><span class="n">_node</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/wrap_util.py:14,</span> in <span class="ni">unary_to_nary.<locals>.nary_operator.<locals>.nary_f.<locals>.unary_f</span><span class="nt">(x)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">12</span> <span class="n">subargs</span> <span class="o">=</span> <span class="n">subvals</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="p">[(</span><span class="n">argnum</span><span class="p">,</span> <span class="n">x</span><span class="p">)])</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">13</span> <span class="k">else</span><span class="p">:</span>
|
||
<span class="ne">---> </span><span class="mi">14</span> <span class="n">subargs</span> <span class="o">=</span> <span class="n">subvals</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="nb">zip</span><span class="p">(</span><span class="n">argnum</span><span class="p">,</span> <span class="n">x</span><span class="p">))</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">15</span> <span class="k">return</span> <span class="n">fun</span><span class="p">(</span><span class="o">*</span><span class="n">subargs</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/util.py:6,</span> in <span class="ni">subvals</span><span class="nt">(x, ivs)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">4</span> <span class="k">def</span> <span class="nf">subvals</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">ivs</span><span class="p">):</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">5</span> <span class="n">x_</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="ne">----> </span><span class="mi">6</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">ivs</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">7</span> <span class="n">x_</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">v</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">8</span> <span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">x_</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/tracer.py:46,</span> in <span class="ni">primitive.<locals>.f_wrapped</span><span class="nt">(*args, **kwargs)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">44</span> <span class="n">ans</span> <span class="o">=</span> <span class="n">f_wrapped</span><span class="p">(</span><span class="o">*</span><span class="n">argvals</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">45</span> <span class="n">node</span> <span class="o">=</span> <span class="n">node_constructor</span><span class="p">(</span><span class="n">ans</span><span class="p">,</span> <span class="n">f_wrapped</span><span class="p">,</span> <span class="n">argvals</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">,</span> <span class="n">argnums</span><span class="p">,</span> <span class="n">parents</span><span class="p">)</span>
|
||
<span class="ne">---> </span><span class="mi">46</span> <span class="k">return</span> <span class="n">new_box</span><span class="p">(</span><span class="n">ans</span><span class="p">,</span> <span class="n">trace</span><span class="p">,</span> <span class="n">node</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">47</span> <span class="k">else</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">48</span> <span class="k">return</span> <span class="n">f_raw</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/tracer.py:120,</span> in <span class="ni">new_box</span><span class="nt">(value, trace, node)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">118</span> <span class="k">return</span> <span class="n">box_type_mappings</span><span class="p">[</span><span class="nb">type</span><span class="p">(</span><span class="n">value</span><span class="p">)](</span><span class="n">value</span><span class="p">,</span> <span class="n">trace</span><span class="p">,</span> <span class="n">node</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">119</span> <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
|
||
<span class="ne">--> </span><span class="mi">120</span> <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"Can't differentiate w.r.t. type </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">value</span><span class="p">)))</span>
|
||
|
||
<span class="ne">TypeError</span>: Can't differentiate w.r.t. type <class 'ellipsis'>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercise-3-gradient-with-one-layer-writing-backpropagation-by-hand">
|
||
<h1>Exercise 3 - Gradient with one layer writing backpropagation by hand<a class="headerlink" href="#exercise-3-gradient-with-one-layer-writing-backpropagation-by-hand" title="Link to this heading">#</a></h1>
|
||
<p>Before you use the gradient you found using autograd, you will have to find the gradient “manually”, to better understand how the backpropagation computation works. To do backpropagation “manually”, you will need to write out expressions for many derivatives along the computation.</p>
|
||
<p>We want to find the gradient of the cost function wrt. the weight and bias. This is quite hard to do directly, so we instead use the chain rule to combine multiple derivatives which are easier to compute.</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{dC}{dW} = \frac{dC}{da}\frac{da}{dz}\frac{dz}{dW}
|
||
\]</div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{dC}{db} = \frac{dC}{da}\frac{da}{dz}\frac{dz}{db}
|
||
\]</div>
|
||
<p><strong>a)</strong> Which intermediary results can be reused between the two expressions?</p>
|
||
<p><strong>b)</strong> What is the derivative of the cost wrt. the final activation? You can use the autograd calculation to make sure you get the correct result. Remember that we compute the mean in mse.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">z</span> <span class="o">=</span> <span class="n">W</span> <span class="o">@</span> <span class="n">x</span> <span class="o">+</span> <span class="n">b</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="n">sigmoid</span><span class="p">(</span><span class="n">z</span><span class="p">)</span>
|
||
|
||
<span class="n">predict</span> <span class="o">=</span> <span class="n">a</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">mse_der</span><span class="p">(</span><span class="n">predict</span><span class="p">,</span> <span class="n">target</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="o">...</span>
|
||
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">mse_der</span><span class="p">(</span><span class="n">predict</span><span class="p">,</span> <span class="n">target</span><span class="p">))</span>
|
||
|
||
<span class="n">cost_autograd</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">mse</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">cost_autograd</span><span class="p">(</span><span class="n">predict</span><span class="p">,</span> <span class="n">target</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>c)</strong> What is the expression for the derivative of the sigmoid activation function? You can use the autograd calculation to make sure you get the correct result.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">sigmoid_der</span><span class="p">(</span><span class="n">z</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="o">...</span>
|
||
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">sigmoid_der</span><span class="p">(</span><span class="n">z</span><span class="p">))</span>
|
||
|
||
<span class="n">sigmoid_autograd</span> <span class="o">=</span> <span class="n">elementwise_grad</span><span class="p">(</span><span class="n">sigmoid</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">sigmoid_autograd</span><span class="p">(</span><span class="n">z</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>d)</strong> Using the two derivatives you just computed, compute this intermetidary gradient you will use later:</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{dC}{dz} = \frac{dC}{da}\frac{da}{dz}
|
||
\]</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">dC_da</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_dz</span> <span class="o">=</span> <span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>e)</strong> What is the derivative of the intermediary z wrt. the weight and bias? What should the shapes be? The one for the weights is a little tricky, it can be easier to play around in the next exercise first. You can also try computing it with autograd to get a hint.</p>
|
||
<p><strong>f)</strong> Now combine the expressions you have worked with so far to compute the gradients! Note that you always need to do a feed forward pass while saving the zs and as before you do backpropagation, as they are used in the derivative expressions</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">dC_da</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_dz</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_dW</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_db</span> <span class="o">=</span> <span class="o">...</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">dC_dW</span><span class="p">,</span> <span class="n">dC_db</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>You should get the same results as with autograd.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">W_g</span><span class="p">,</span> <span class="n">b_g</span> <span class="o">=</span> <span class="n">autograd_one_layer</span><span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">W_g</span><span class="p">,</span> <span class="n">b_g</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercise-4-gradient-with-two-layers-writing-backpropagation-by-hand">
|
||
<h1>Exercise 4 - Gradient with two layers writing backpropagation by hand<a class="headerlink" href="#exercise-4-gradient-with-two-layers-writing-backpropagation-by-hand" title="Link to this heading">#</a></h1>
|
||
<p>Now that you have implemented backpropagation for one layer, you have found most of the expressions you will need for more layers. Let’s move up to two layers.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
|
||
<span class="n">target</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">4</span><span class="p">)</span>
|
||
|
||
<span class="n">W1</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
|
||
<span class="n">b1</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span>
|
||
|
||
<span class="n">W2</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
|
||
<span class="n">b2</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">4</span><span class="p">)</span>
|
||
|
||
<span class="n">layers</span> <span class="o">=</span> <span class="p">[(</span><span class="n">W1</span><span class="p">,</span> <span class="n">b1</span><span class="p">),</span> <span class="p">(</span><span class="n">W2</span><span class="p">,</span> <span class="n">b2</span><span class="p">)]</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">z1</span> <span class="o">=</span> <span class="n">W1</span> <span class="o">@</span> <span class="n">x</span> <span class="o">+</span> <span class="n">b1</span>
|
||
<span class="n">a1</span> <span class="o">=</span> <span class="n">sigmoid</span><span class="p">(</span><span class="n">z1</span><span class="p">)</span>
|
||
<span class="n">z2</span> <span class="o">=</span> <span class="n">W2</span> <span class="o">@</span> <span class="n">a1</span> <span class="o">+</span> <span class="n">b2</span>
|
||
<span class="n">a2</span> <span class="o">=</span> <span class="n">sigmoid</span><span class="p">(</span><span class="n">z2</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We begin by computing the gradients of the last layer, as the gradients must be propagated backwards from the end.</p>
|
||
<p><strong>a)</strong> Compute the gradients of the last layer, just like you did the single layer in the previous exercise.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">dC_da2</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_dz2</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_dW2</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_db2</span> <span class="o">=</span> <span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>To find the derivative of the cost wrt. the activation of the first layer, we need a new expression, the one furthest to the right in the following.</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{dC}{da_1} = \frac{dC}{dz_2}\frac{dz_2}{da_1}
|
||
\]</div>
|
||
<p><strong>b)</strong> What is the derivative of the second layer intermetiate wrt. the first layer activation? (First recall how you compute <span class="math notranslate nohighlight">\(z_2\)</span>)</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{dz_2}{da_1}
|
||
\]</div>
|
||
<p><strong>c)</strong> Use this expression, together with expressions which are equivelent to ones for the last layer to compute all the derivatives of the first layer.</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{dC}{dW_1} = \frac{dC}{da_1}\frac{da_1}{dz_1}\frac{dz_1}{dW_1}
|
||
\]</div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{dC}{db_1} = \frac{dC}{da_1}\frac{da_1}{dz_1}\frac{dz_1}{db_1}
|
||
\]</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">dC_da1</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_dz1</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_dW1</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_db1</span> <span class="o">=</span> <span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="nb">print</span><span class="p">(</span><span class="n">dC_dW1</span><span class="p">,</span> <span class="n">dC_db1</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">dC_dW2</span><span class="p">,</span> <span class="n">dC_db2</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>d)</strong> Make sure you got the same gradient as the following code which uses autograd to do backpropagation.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">feed_forward_two_layers</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
|
||
<span class="n">W1</span><span class="p">,</span> <span class="n">b1</span> <span class="o">=</span> <span class="n">layers</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||
<span class="n">z1</span> <span class="o">=</span> <span class="n">W1</span> <span class="o">@</span> <span class="n">x</span> <span class="o">+</span> <span class="n">b1</span>
|
||
<span class="n">a1</span> <span class="o">=</span> <span class="n">sigmoid</span><span class="p">(</span><span class="n">z1</span><span class="p">)</span>
|
||
|
||
<span class="n">W2</span><span class="p">,</span> <span class="n">b2</span> <span class="o">=</span> <span class="n">layers</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
||
<span class="n">z2</span> <span class="o">=</span> <span class="n">W2</span> <span class="o">@</span> <span class="n">a1</span> <span class="o">+</span> <span class="n">b2</span>
|
||
<span class="n">a2</span> <span class="o">=</span> <span class="n">sigmoid</span><span class="p">(</span><span class="n">z2</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">a2</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">cost_two_layers</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">target</span><span class="p">):</span>
|
||
<span class="n">predict</span> <span class="o">=</span> <span class="n">feed_forward_two_layers</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">mse</span><span class="p">(</span><span class="n">predict</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
|
||
|
||
|
||
<span class="n">grad_two_layers</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">cost_two_layers</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||
<span class="n">grad_two_layers</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>e)</strong> How would you use the gradient from this layer to compute the gradient of an even earlier layer? Would the expressions be any different?</p>
|
||
</section>
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercise-5-gradient-with-any-number-of-layers-writing-backpropagation-by-hand">
|
||
<h1>Exercise 5 - Gradient with any number of layers writing backpropagation by hand<a class="headerlink" href="#exercise-5-gradient-with-any-number-of-layers-writing-backpropagation-by-hand" title="Link to this heading">#</a></h1>
|
||
<p>Well done on getting this far! Now it’s time to compute the gradient with any number of layers.</p>
|
||
<p>First, some code from the general neural network code from last week. Note that we are still sending in one input vector at a time. We will change it to use batched inputs later.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">create_layers</span><span class="p">(</span><span class="n">network_input_size</span><span class="p">,</span> <span class="n">layer_output_sizes</span><span class="p">):</span>
|
||
<span class="n">layers</span> <span class="o">=</span> <span class="p">[]</span>
|
||
|
||
<span class="n">i_size</span> <span class="o">=</span> <span class="n">network_input_size</span>
|
||
<span class="k">for</span> <span class="n">layer_output_size</span> <span class="ow">in</span> <span class="n">layer_output_sizes</span><span class="p">:</span>
|
||
<span class="n">W</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">layer_output_size</span><span class="p">,</span> <span class="n">i_size</span><span class="p">)</span>
|
||
<span class="n">b</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">layer_output_size</span><span class="p">)</span>
|
||
<span class="n">layers</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">))</span>
|
||
|
||
<span class="n">i_size</span> <span class="o">=</span> <span class="n">layer_output_size</span>
|
||
<span class="k">return</span> <span class="n">layers</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">feed_forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">):</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="nb">input</span>
|
||
<span class="k">for</span> <span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">),</span> <span class="n">activation_func</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">):</span>
|
||
<span class="n">z</span> <span class="o">=</span> <span class="n">W</span> <span class="o">@</span> <span class="n">a</span> <span class="o">+</span> <span class="n">b</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="n">activation_func</span><span class="p">(</span><span class="n">z</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">a</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">cost</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="nb">input</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">,</span> <span class="n">target</span><span class="p">):</span>
|
||
<span class="n">predict</span> <span class="o">=</span> <span class="n">feed_forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">mse</span><span class="p">(</span><span class="n">predict</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>You might have already have noticed a very important detail in backpropagation: You need the values from the forward pass to compute all the gradients! The feed forward method above is great for efficiency and for using autograd, as it only cares about computing the final output, but now we need to also save the results along the way.</p>
|
||
<p>Here is a function which does that for you.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">feed_forward_saver</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">):</span>
|
||
<span class="n">layer_inputs</span> <span class="o">=</span> <span class="p">[]</span>
|
||
<span class="n">zs</span> <span class="o">=</span> <span class="p">[]</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="nb">input</span>
|
||
<span class="k">for</span> <span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">),</span> <span class="n">activation_func</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">):</span>
|
||
<span class="n">layer_inputs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">a</span><span class="p">)</span>
|
||
<span class="n">z</span> <span class="o">=</span> <span class="n">W</span> <span class="o">@</span> <span class="n">a</span> <span class="o">+</span> <span class="n">b</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="n">activation_func</span><span class="p">(</span><span class="n">z</span><span class="p">)</span>
|
||
|
||
<span class="n">zs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">z</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">layer_inputs</span><span class="p">,</span> <span class="n">zs</span><span class="p">,</span> <span class="n">a</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>a)</strong> Now, complete the backpropagation function so that it returns the gradient of the cost function wrt. all the weigths and biases. Use the autograd calculation below to make sure you get the correct answer.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">backpropagation</span><span class="p">(</span>
|
||
<span class="nb">input</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">activation_ders</span><span class="p">,</span> <span class="n">cost_der</span><span class="o">=</span><span class="n">mse_der</span>
|
||
<span class="p">):</span>
|
||
<span class="n">layer_inputs</span><span class="p">,</span> <span class="n">zs</span><span class="p">,</span> <span class="n">predict</span> <span class="o">=</span> <span class="n">feed_forward_saver</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">)</span>
|
||
|
||
<span class="n">layer_grads</span> <span class="o">=</span> <span class="p">[()</span> <span class="k">for</span> <span class="n">layer</span> <span class="ow">in</span> <span class="n">layers</span><span class="p">]</span>
|
||
|
||
<span class="c1"># We loop over the layers, from the last to the first</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">reversed</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">layers</span><span class="p">))):</span>
|
||
<span class="n">layer_input</span><span class="p">,</span> <span class="n">z</span><span class="p">,</span> <span class="n">activation_der</span> <span class="o">=</span> <span class="n">layer_inputs</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">zs</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">activation_ders</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||
|
||
<span class="k">if</span> <span class="n">i</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">layers</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">:</span>
|
||
<span class="c1"># For last layer we use cost derivative as dC_da(L) can be computed directly</span>
|
||
<span class="n">dC_da</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="c1"># For other layers we build on previous z derivative, as dC_da(i) = dC_dz(i+1) * dz(i+1)_da(i)</span>
|
||
<span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">)</span> <span class="o">=</span> <span class="n">layers</span><span class="p">[</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span>
|
||
<span class="n">dC_da</span> <span class="o">=</span> <span class="o">...</span>
|
||
|
||
<span class="n">dC_dz</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_dW</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">dC_db</span> <span class="o">=</span> <span class="o">...</span>
|
||
|
||
<span class="n">layer_grads</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">dC_dW</span><span class="p">,</span> <span class="n">dC_db</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">layer_grads</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">network_input_size</span> <span class="o">=</span> <span class="mi">2</span>
|
||
<span class="n">layer_output_sizes</span> <span class="o">=</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]</span>
|
||
<span class="n">activation_funcs</span> <span class="o">=</span> <span class="p">[</span><span class="n">sigmoid</span><span class="p">,</span> <span class="n">ReLU</span><span class="p">]</span>
|
||
<span class="n">activation_ders</span> <span class="o">=</span> <span class="p">[</span><span class="n">sigmoid_der</span><span class="p">,</span> <span class="n">ReLU_der</span><span class="p">]</span>
|
||
|
||
<span class="n">layers</span> <span class="o">=</span> <span class="n">create_layers</span><span class="p">(</span><span class="n">network_input_size</span><span class="p">,</span> <span class="n">layer_output_sizes</span><span class="p">)</span>
|
||
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">network_input_size</span><span class="p">)</span>
|
||
<span class="n">target</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">4</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">layer_grads</span> <span class="o">=</span> <span class="n">backpropagation</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">activation_ders</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">layer_grads</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">cost_grad</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">cost</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||
<span class="n">cost_grad</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="p">[</span><span class="n">sigmoid</span><span class="p">,</span> <span class="n">ReLU</span><span class="p">],</span> <span class="n">target</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercise-6-batched-inputs">
|
||
<h1>Exercise 6 - Batched inputs<a class="headerlink" href="#exercise-6-batched-inputs" title="Link to this heading">#</a></h1>
|
||
<p>Make new versions of all the functions in exercise 5 which now take batched inputs instead. See last weeks exercise 5 for details on how to batch inputs to neural networks. You will also need to update the backpropogation function.</p>
|
||
</section>
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercise-7-training">
|
||
<h1>Exercise 7 - Training<a class="headerlink" href="#exercise-7-training" title="Link to this heading">#</a></h1>
|
||
<p><strong>a)</strong> Complete exercise 6 and 7 from last week, but use your own backpropogation implementation to compute the gradient.</p>
|
||
<p><strong>b)</strong> Use stochastic gradient descent with momentum when you train your network.</p>
|
||
</section>
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercise-8-optional-object-orientation">
|
||
<h1>Exercise 8 (Optional) - Object orientation<a class="headerlink" href="#exercise-8-optional-object-orientation" title="Link to this heading">#</a></h1>
|
||
<p>Passing in the layers, activations functions, activation derivatives and cost derivatives into the functions each time leads to code which is easy to understand in isoloation, but messier when used in a larger context with data splitting, data scaling, gradient methods and so forth. Creating an object which stores these values can lead to code which is much easier to use.</p>
|
||
<p><strong>a)</strong> Write a neural network class. You are free to implement it how you see fit, though we strongly recommend to not save any input or output values as class attributes, nor let the neural network class handle gradient methods internally. Gradient methods should be handled outside, by performing general operations on the layer_grads list using functions or classes separate to the neural network.</p>
|
||
<p>We provide here a skeleton structure which should get you started.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">NeuralNetwork</span><span class="p">:</span>
|
||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
|
||
<span class="bp">self</span><span class="p">,</span>
|
||
<span class="n">network_input_size</span><span class="p">,</span>
|
||
<span class="n">layer_output_sizes</span><span class="p">,</span>
|
||
<span class="n">activation_funcs</span><span class="p">,</span>
|
||
<span class="n">activation_ders</span><span class="p">,</span>
|
||
<span class="n">cost_fun</span><span class="p">,</span>
|
||
<span class="n">cost_der</span><span class="p">,</span>
|
||
<span class="p">):</span>
|
||
<span class="k">pass</span>
|
||
|
||
<span class="k">def</span> <span class="nf">predict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inputs</span><span class="p">):</span>
|
||
<span class="c1"># Simple feed forward pass</span>
|
||
<span class="k">pass</span>
|
||
|
||
<span class="k">def</span> <span class="nf">cost</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">targets</span><span class="p">):</span>
|
||
<span class="k">pass</span>
|
||
|
||
<span class="k">def</span> <span class="nf">_feed_forward_saver</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inputs</span><span class="p">):</span>
|
||
<span class="k">pass</span>
|
||
|
||
<span class="k">def</span> <span class="nf">compute_gradient</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">targets</span><span class="p">):</span>
|
||
<span class="k">pass</span>
|
||
|
||
<span class="k">def</span> <span class="nf">update_weights</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">layer_grads</span><span class="p">):</span>
|
||
<span class="k">pass</span>
|
||
|
||
<span class="c1"># These last two methods are not needed in the project, but they can be nice to have! The first one has a layers parameter so that you can use autograd on it</span>
|
||
<span class="k">def</span> <span class="nf">autograd_compliant_predict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">inputs</span><span class="p">):</span>
|
||
<span class="k">pass</span>
|
||
|
||
<span class="k">def</span> <span class="nf">autograd_gradient</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">targets</span><span class="p">):</span>
|
||
<span class="k">pass</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
name: "python3",
|
||
path: "./."
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</article>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<footer class="prev-next-footer d-print-none">
|
||
|
||
<div class="prev-next-area">
|
||
<a class="left-prev"
|
||
href="week43.html"
|
||
title="previous page">
|
||
<i class="fa-solid fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title">Week 43: Deep Learning: Constructing a Neural Network code and solving differential equations</p>
|
||
</div>
|
||
</a>
|
||
<a class="right-next"
|
||
href="week44.html"
|
||
title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title">Week 44, Convolutional Neural Networks (CNN)</p>
|
||
</div>
|
||
<i class="fa-solid fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<dialog id="pst-secondary-sidebar-modal"></dialog>
|
||
<div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
|
||
|
||
|
||
<div class="sidebar-secondary-item">
|
||
<div class="page-toc tocsection onthispage">
|
||
<i class="fa-solid fa-list"></i> Contents
|
||
</div>
|
||
<nav class="bd-toc-nav page-toc">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#">Exercises week 43</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#overarching-aims-of-the-exercises-this-week">Overarching aims of the exercises this week</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-1-understand-the-feed-forward-pass">Exercise 1 - Understand the feed forward pass</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-2-gradient-with-one-layer-using-autograd">Exercise 2 - Gradient with one layer using autograd</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-3-gradient-with-one-layer-writing-backpropagation-by-hand">Exercise 3 - Gradient with one layer writing backpropagation by hand</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-4-gradient-with-two-layers-writing-backpropagation-by-hand">Exercise 4 - Gradient with two layers writing backpropagation by hand</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-5-gradient-with-any-number-of-layers-writing-backpropagation-by-hand">Exercise 5 - Gradient with any number of layers writing backpropagation by hand</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-6-batched-inputs">Exercise 6 - Batched inputs</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-7-training">Exercise 7 - Training</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-8-optional-object-orientation">Exercise 8 (Optional) - Object orientation</a></li>
|
||
</ul>
|
||
|
||
</nav></div>
|
||
|
||
</div></div>
|
||
|
||
|
||
</div>
|
||
<footer class="bd-footer-content">
|
||
|
||
<div class="bd-footer-content__inner container">
|
||
|
||
<div class="footer-item">
|
||
|
||
<p class="component-author">
|
||
By Morten Hjorth-Jensen
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
|
||
<p class="copyright">
|
||
|
||
© Copyright 2023.
|
||
<br/>
|
||
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</footer>
|
||
|
||
|
||
</main>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
||
<script defer src="_static/scripts/bootstrap.js?digest=26a4bc78f4c0ddb94549"></script>
|
||
<script defer src="_static/scripts/pydata-sphinx-theme.js?digest=26a4bc78f4c0ddb94549"></script>
|
||
|
||
<footer class="bd-footer">
|
||
</footer>
|
||
</body>
|
||
</html> |