Morten Hjorth-Jensen
968 lines
66 KiB
HTML
968 lines
66 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
|
||
<html lang="en" data-content_root="./" >
|
||
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<title>Exercises week 41 — Applied Data Analysis and Machine Learning</title>
|
||
|
||
|
||
|
||
<script data-cfasync="false">
|
||
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
||
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
||
</script>
|
||
|
||
<!-- Loaded before other Sphinx assets -->
|
||
<link href="_static/styles/theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="_static/styles/bootstrap.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="_static/styles/pydata-sphinx-theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
|
||
|
||
<link href="_static/vendor/fontawesome/6.5.2/css/all.min.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-solid-900.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-brands-400.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-regular-400.woff2" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=fa44fd50" />
|
||
<link rel="stylesheet" type="text/css" href="_static/styles/sphinx-book-theme.css?v=eba8b062" />
|
||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css?v=13237357" />
|
||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css?v=76b2166b" />
|
||
<link rel="stylesheet" type="text/css" href="_static/mystnb.8ecb98da25f57f5357bf6f572d296f466b2cfe2517ffebfabe82451661e28f02.css?v=6644e6bb" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css?v=4fa983c6" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-design.min.css?v=95c83b7e" />
|
||
|
||
<!-- Pre-loaded scripts that we'll load fully later -->
|
||
<link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<script src="_static/vendor/fontawesome/6.5.2/js/all.min.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<script src="_static/documentation_options.js?v=9eb32ce0"></script>
|
||
<script src="_static/doctools.js?v=9a2dae69"></script>
|
||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
<script src="_static/clipboard.min.js?v=a7894cd8"></script>
|
||
<script src="_static/copybutton.js?v=f281be69"></script>
|
||
<script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="_static/togglebutton.js?v=4a39c7ea"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script src="_static/design-tabs.js?v=f930bc37"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script async="async" src="_static/sphinx-thebe.js?v=c100c467"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
||
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
<script>DOCUMENTATION_OPTIONS.pagename = 'exercisesweek41';</script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="Week 42 Constructing a Neural Network code with examples" href="week42.html" />
|
||
<link rel="prev" title="Week 41 Neural networks and constructing a neural network code" href="week41.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||
<meta name="docsearch:language" content="en"/>
|
||
</head>
|
||
|
||
|
||
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
||
|
||
|
||
|
||
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
||
|
||
<div id="pst-scroll-pixel-helper"></div>
|
||
|
||
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
||
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
||
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-primary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-primary" for="pst-primary-sidebar-checkbox"></label>
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-secondary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-secondary" for="pst-secondary-sidebar-checkbox"></label>
|
||
|
||
<div class="search-button__wrapper">
|
||
<div class="search-button__overlay"></div>
|
||
<div class="search-button__search-container">
|
||
<form class="bd-search d-flex align-items-center"
|
||
action="search.html"
|
||
method="get">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<input type="search"
|
||
class="form-control"
|
||
name="q"
|
||
id="search-input"
|
||
placeholder="Search this book..."
|
||
aria-label="Search this book..."
|
||
autocomplete="off"
|
||
autocorrect="off"
|
||
autocapitalize="off"
|
||
spellcheck="false"/>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
||
</form></div>
|
||
</div>
|
||
|
||
<div class="pst-async-banner-revealer d-none">
|
||
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
||
</div>
|
||
|
||
|
||
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
||
</header>
|
||
|
||
|
||
<div class="bd-container">
|
||
<div class="bd-container__inner bd-page-width">
|
||
|
||
|
||
|
||
<div class="bd-sidebar-primary bd-sidebar">
|
||
|
||
|
||
|
||
<div class="sidebar-header-items sidebar-primary__section">
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
<div class="sidebar-primary-items__start sidebar-primary__section">
|
||
<div class="sidebar-primary-item">
|
||
|
||
|
||
|
||
|
||
|
||
<a class="navbar-brand logo" href="intro.html">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<img src="_static/logo.png" class="logo__image only-light" alt="Applied Data Analysis and Machine Learning - Home"/>
|
||
<script>document.write(`<img src="_static/logo.png" class="logo__image only-dark" alt="Applied Data Analysis and Machine Learning - Home"/>`);</script>
|
||
|
||
|
||
</a></div>
|
||
<div class="sidebar-primary-item">
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<span class="search-button__default-text">Search</span>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||
</button>
|
||
`);
|
||
</script></div>
|
||
<div class="sidebar-primary-item"><nav class="bd-links bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item navbar-nav active">
|
||
|
||
<ul class="nav bd-sidenav bd-sidenav__home-link">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="intro.html">
|
||
Applied Data Analysis and Machine Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">About the course</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="schedule.html">Course setting</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="teachers.html">Teachers and Grading</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="textbooks.html">Textbooks</a></li>
|
||
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Review of Statistics with Resampling Techniques and Linear Algebra</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="statistics.html">1. Elements of Probability Theory and Statistical Data Analysis</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="linalg.html">2. Linear Algebra, Handling of Arrays and more Python Features</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">From Regression to Support Vector Machines</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter1.html">3. Linear Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter2.html">4. Ridge and Lasso Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter3.html">5. Resampling Methods</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter4.html">6. Logistic Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapteroptimization.html">7. Optimization, the central part of any Machine Learning algortithm</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter5.html">8. Support Vector Machines, overarching aims</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Decision Trees, Ensemble Methods and Boosting</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter6.html">9. Decision trees, overarching aims</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter7.html">10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Dimensionality Reduction</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter8.html">11. Basic ideas of the Principal Component Analysis (PCA)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="clustering.html">12. Clustering and Unsupervised Learning</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Deep Learning Methods</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter9.html">13. Neural networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter10.html">14. Building a Feed Forward Neural Network</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter11.html">15. Solving Differential Equations with Deep Learning</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter12.html">16. Convolutional Neural Networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter13.html">17. Recurrent neural networks: Overarching view</a></li>
|
||
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Weekly material, notes and exercises</span></p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek34.html">Exercises week 34</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week34.html">Week 34: Introduction to the course, Logistics and Practicalities</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek35.html">Exercises week 35</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week35.html">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Gradient descent methods</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 38</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week38.html">Week 38: Statistical analysis, bias-variance tradeoff and resampling methods</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek39.html">Exercises week 39</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week39.html">Week 39: Resampling methods and logistic regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week40.html">Week 40: Gradient descent methods (continued) and start Neural networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week41.html">Week 41 Neural networks and constructing a neural network code</a></li>
|
||
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Exercises week 41</a></li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="toctree-l1"><a class="reference internal" href="week42.html">Week 42 Constructing a Neural Network code with examples</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek42.html">Exercises week 42</a></li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="project1.html">Project 1 on Machine Learning, deadline October 6 (midnight), 2025</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="project2.html">Project 2 on Machine Learning, deadline November 10 (Midnight)</a></li>
|
||
</ul>
|
||
|
||
</div>
|
||
</nav></div>
|
||
</div>
|
||
|
||
|
||
<div class="sidebar-primary-items__end sidebar-primary__section">
|
||
</div>
|
||
|
||
<div id="rtd-footer-container"></div>
|
||
|
||
|
||
</div>
|
||
|
||
<main id="main-content" class="bd-main" role="main">
|
||
|
||
|
||
|
||
<div class="sbt-scroll-pixel-helper"></div>
|
||
|
||
<div class="bd-content">
|
||
<div class="bd-article-container">
|
||
|
||
<div class="bd-header-article d-print-none">
|
||
<div class="header-article-items header-article__inner">
|
||
|
||
<div class="header-article-items__start">
|
||
|
||
<div class="header-article-item"><button class="sidebar-toggle primary-toggle btn btn-sm" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-bars"></span>
|
||
</button></div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="header-article-items__end">
|
||
|
||
<div class="header-article-item">
|
||
|
||
<div class="article-header-buttons">
|
||
|
||
|
||
|
||
|
||
|
||
<div class="dropdown dropdown-download-buttons">
|
||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
|
||
<i class="fas fa-download"></i>
|
||
</button>
|
||
<ul class="dropdown-menu">
|
||
|
||
|
||
|
||
<li><a href="_sources/exercisesweek41.ipynb" target="_blank"
|
||
class="btn btn-sm btn-download-source-button dropdown-item"
|
||
title="Download source file"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file"></i>
|
||
</span>
|
||
<span class="btn__text-container">.ipynb</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li>
|
||
<button onclick="window.print()"
|
||
class="btn btn-sm btn-download-pdf-button dropdown-item"
|
||
title="Print to PDF"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file-pdf"></i>
|
||
</span>
|
||
<span class="btn__text-container">.pdf</span>
|
||
</button>
|
||
</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<button onclick="toggleFullScreen()"
|
||
class="btn btn-sm btn-fullscreen-button"
|
||
title="Fullscreen mode"
|
||
data-bs-placement="bottom" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-expand"></i>
|
||
</span>
|
||
|
||
</button>
|
||
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light"></i>
|
||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark"></i>
|
||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm pst-navbar-icon search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass fa-lg"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
<button class="sidebar-toggle secondary-toggle btn btn-sm" title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-list"></span>
|
||
</button>
|
||
</div></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>Exercises week 41</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#">Exercises week 41</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#overarching-aims-of-the-exercises-this-week">Overarching aims of the exercises this week</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-1">Exercise 1</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-2">Exercise 2</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-3">Exercise 3</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-4-custom-activation-for-each-layer">Exercise 4 - Custom activation for each layer</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-5-processing-multiple-inputs-at-once">Exercise 5 - Processing multiple inputs at once</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-6-predicting-on-real-data">Exercise 6 - Predicting on real data</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-7-training-on-real-data-optional">Exercise 7 - Training on real data (Optional)</a></li>
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="searchbox"></div>
|
||
<article class="bd-article">
|
||
|
||
<!-- HTML file automatically generated from DocOnce source (https://github.com/doconce/doconce/)
|
||
doconce format html exercisesweek41.do.txt -->
|
||
<!-- dom:TITLE: Exercises week 41 -->
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercises-week-41">
|
||
<h1>Exercises week 41<a class="headerlink" href="#exercises-week-41" title="Link to this heading">#</a></h1>
|
||
<p><strong>October 6-10, 2025</strong></p>
|
||
<p>Date: <strong>Deadline is Friday October 10 at midnight</strong></p>
|
||
</section>
|
||
<section class="tex2jax_ignore mathjax_ignore" id="overarching-aims-of-the-exercises-this-week">
|
||
<h1>Overarching aims of the exercises this week<a class="headerlink" href="#overarching-aims-of-the-exercises-this-week" title="Link to this heading">#</a></h1>
|
||
<p>This week, you will implement the entire feed-forward pass of a neural network! Next week you will compute the gradient of the network by implementing back-propagation manually, and by using autograd which does back-propagation for you (much easier!). Next week, you will also use the gradient to optimize the network with a gradient method! However, there is an optional exercise this week to get started on training the network and getting good results!</p>
|
||
<p>We recommend that you do the exercises this week by editing and running this notebook file, as it includes some checks along the way that you have implemented the pieces of the feed-forward pass correctly, and running small parts of the code at a time will be important for understanding the methods.</p>
|
||
<p>If you have trouble running a notebook, you can run this notebook in google colab instead (<a class="reference external" href="https://colab.research.google.com/drive/1zKibVQf-iAYaAn2-GlKfgRjHtLnPlBX4#offline=true&amp;sandboxMode=true">https://colab.research.google.com/drive/1zKibVQf-iAYaAn2-GlKfgRjHtLnPlBX4#offline=true&sandboxMode=true</a>), an updated link will be provided on the course discord (you can also send an email to <a class="reference external" href="mailto:k.h.fredly%40fys.uio.no">k<span>.</span>h<span>.</span>fredly<span>@</span>fys<span>.</span>uio<span>.</span>no</a> if you encounter any trouble), though we recommend that you set up VSCode and your python environment to run code like this locally.</p>
|
||
<p>First, here are some functions you are going to need, don’t change this cell. If you are unable to import autograd, just swap in normal numpy until you want to do the final optional exercise.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">autograd.numpy</span> <span class="k">as</span> <span class="nn">np</span> <span class="c1"># We need to use this numpy wrapper to make automatic differentiation work later</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn</span> <span class="kn">import</span> <span class="n">datasets</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">accuracy_score</span>
|
||
|
||
|
||
<span class="c1"># Defining some activation functions</span>
|
||
<span class="k">def</span> <span class="nf">ReLU</span><span class="p">(</span><span class="n">z</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">where</span><span class="p">(</span><span class="n">z</span> <span class="o">></span> <span class="mi">0</span><span class="p">,</span> <span class="n">z</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">sigmoid</span><span class="p">(</span><span class="n">z</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="mi">1</span> <span class="o">/</span> <span class="p">(</span><span class="mi">1</span> <span class="o">+</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="n">z</span><span class="p">))</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">softmax</span><span class="p">(</span><span class="n">z</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""Compute softmax values for each set of scores in the rows of the matrix z.</span>
|
||
<span class="sd"> Used with batched input data."""</span>
|
||
<span class="n">e_z</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="n">z</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">z</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">))</span>
|
||
<span class="k">return</span> <span class="n">e_z</span> <span class="o">/</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">e_z</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)[:,</span> <span class="n">np</span><span class="o">.</span><span class="n">newaxis</span><span class="p">]</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">softmax_vec</span><span class="p">(</span><span class="n">z</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""Compute softmax values for each set of scores in the vector z.</span>
|
||
<span class="sd"> Use this function when you use the activation function on one vector at a time"""</span>
|
||
<span class="n">e_z</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="n">z</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">z</span><span class="p">))</span>
|
||
<span class="k">return</span> <span class="n">e_z</span> <span class="o">/</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">e_z</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercise-1">
|
||
<h1>Exercise 1<a class="headerlink" href="#exercise-1" title="Link to this heading">#</a></h1>
|
||
<p>In this exercise you will compute the activation of the first layer. You only need to change the code in the cells right below an exercise, the rest works out of the box. Feel free to make changes and see how stuff works though!</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">2024</span><span class="p">)</span>
|
||
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span> <span class="c1"># network input. This is a single input with two features</span>
|
||
<span class="n">W1</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span> <span class="c1"># first layer weights</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>a)</strong> Given the shape of the first layer weight matrix, what is the input shape of the neural network? What is the output shape of the first layer?</p>
|
||
<p><strong>b)</strong> Define the bias of the first layer, <code class="docutils literal notranslate"><span class="pre">b1</span></code>with the correct shape. (Run the next cell right after the previous to get the random generated values to line up with the test solution below)</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">b1</span> <span class="o">=</span> <span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>c)</strong> Compute the intermediary <code class="docutils literal notranslate"><span class="pre">z1</span></code> for the first layer</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">z1</span> <span class="o">=</span> <span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>d)</strong> Compute the activation <code class="docutils literal notranslate"><span class="pre">a1</span></code> for the first layer using the ReLU activation function defined earlier.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">a1</span> <span class="o">=</span> <span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Confirm that you got the correct activation with the test below. Make sure that you define <code class="docutils literal notranslate"><span class="pre">b1</span></code> with the randn function right after you define <code class="docutils literal notranslate"><span class="pre">W1</span></code>.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">sol1</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">0.60610368</span><span class="p">,</span> <span class="mf">4.0076268</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="mf">0.56469864</span><span class="p">])</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">allclose</span><span class="p">(</span><span class="n">a1</span><span class="p">,</span> <span class="n">sol1</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercise-2">
|
||
<h1>Exercise 2<a class="headerlink" href="#exercise-2" title="Link to this heading">#</a></h1>
|
||
<p>Now we will add a layer to the network with an output of length 8 and ReLU activation.</p>
|
||
<p><strong>a)</strong> What is the input of the second layer? What is its shape?</p>
|
||
<p><strong>b)</strong> Define the weight and bias of the second layer with the right shapes.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">W2</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">b2</span> <span class="o">=</span> <span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>c)</strong> Compute the intermediary <code class="docutils literal notranslate"><span class="pre">z2</span></code> and activation <code class="docutils literal notranslate"><span class="pre">a2</span></code> for the second layer.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">z2</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">a2</span> <span class="o">=</span> <span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Confirm that you got the correct activation shape with the test below.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="nb">print</span><span class="p">(</span>
|
||
<span class="n">np</span><span class="o">.</span><span class="n">allclose</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">a2</span><span class="p">)),</span> <span class="mf">2980.9579870417283</span><span class="p">)</span>
|
||
<span class="p">)</span> <span class="c1"># This should evaluate to True if a2 has the correct shape :)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercise-3">
|
||
<h1>Exercise 3<a class="headerlink" href="#exercise-3" title="Link to this heading">#</a></h1>
|
||
<p>We often want our neural networks to have many layers of varying sizes. To avoid writing very long and error-prone code where we explicitly define and evaluate each layer we should keep all our layers in a single variable which is easy to create and use.</p>
|
||
<p><strong>a)</strong> Complete the function below so that it returns a list <code class="docutils literal notranslate"><span class="pre">layers</span></code> of weight and bias tuples <code class="docutils literal notranslate"><span class="pre">(W,</span> <span class="pre">b)</span></code> for each layer, in order, with the correct shapes that we can use later as our network parameters.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">create_layers</span><span class="p">(</span><span class="n">network_input_size</span><span class="p">,</span> <span class="n">layer_output_sizes</span><span class="p">):</span>
|
||
<span class="n">layers</span> <span class="o">=</span> <span class="p">[]</span>
|
||
|
||
<span class="n">i_size</span> <span class="o">=</span> <span class="n">network_input_size</span>
|
||
<span class="k">for</span> <span class="n">layer_output_size</span> <span class="ow">in</span> <span class="n">layer_output_sizes</span><span class="p">:</span>
|
||
<span class="n">W</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">b</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">layers</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">))</span>
|
||
|
||
<span class="n">i_size</span> <span class="o">=</span> <span class="n">layer_output_size</span>
|
||
<span class="k">return</span> <span class="n">layers</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>b)</strong> Comple the function below so that it evaluates the intermediary <code class="docutils literal notranslate"><span class="pre">z</span></code> and activation <code class="docutils literal notranslate"><span class="pre">a</span></code> for each layer, with ReLU actication, and returns the final activation <code class="docutils literal notranslate"><span class="pre">a</span></code>. This is the complete feed-forward pass, a full neural network!</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">feed_forward_all_relu</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="nb">input</span><span class="p">):</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="nb">input</span>
|
||
<span class="k">for</span> <span class="n">W</span><span class="p">,</span> <span class="n">b</span> <span class="ow">in</span> <span class="n">layers</span><span class="p">:</span>
|
||
<span class="n">z</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="k">return</span> <span class="n">a</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>c)</strong> Create a network with input size 8 and layers with output sizes 10, 16, 6, 2. Evaluate it and make sure that you get the correct size vectors along the way.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">input_size</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">layer_output_sizes</span> <span class="o">=</span> <span class="p">[</span><span class="o">...</span><span class="p">]</span>
|
||
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">input_size</span><span class="p">)</span>
|
||
<span class="n">layers</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">predict</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">predict</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>d)</strong> Why is a neural network with no activation functions mathematically equivelent to(can be reduced to) a neural network with only one layer?</p>
|
||
</section>
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercise-4-custom-activation-for-each-layer">
|
||
<h1>Exercise 4 - Custom activation for each layer<a class="headerlink" href="#exercise-4-custom-activation-for-each-layer" title="Link to this heading">#</a></h1>
|
||
<p>So far, every layer has used the same activation, ReLU. We often want to use other types of activation however, so we need to update our code to support multiple types of activation functions. Make sure that you have completed every previous exercise before trying this one.</p>
|
||
<p><strong>a)</strong> Complete the <code class="docutils literal notranslate"><span class="pre">feed_forward</span></code> function which accepts a list of activation functions as an argument, and which evaluates these activation functions at each layer.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">feed_forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">):</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="nb">input</span>
|
||
<span class="k">for</span> <span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">),</span> <span class="n">activation_func</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">):</span>
|
||
<span class="n">z</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="k">return</span> <span class="n">a</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>b)</strong> You are now given a list with three activation functions, two ReLU and one sigmoid. (Don’t call them yet! you can make a list with function names as elements, and then call these elements of the list later. If you add other functions than the ones defined at the start of the notebook, make sure everything is defined using autograd’s numpy wrapper, like above, since we want to use automatic differentiation on all of these functions later.)</p>
|
||
<p>Evaluate a network with three layers and these activation functions.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">network_input_size</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">layer_output_sizes</span> <span class="o">=</span> <span class="p">[</span><span class="o">...</span><span class="p">]</span>
|
||
<span class="n">activation_funcs</span> <span class="o">=</span> <span class="p">[</span><span class="n">ReLU</span><span class="p">,</span> <span class="n">ReLU</span><span class="p">,</span> <span class="n">sigmoid</span><span class="p">]</span>
|
||
<span class="n">layers</span> <span class="o">=</span> <span class="o">...</span>
|
||
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">network_input_size</span><span class="p">)</span>
|
||
<span class="n">feed_forward</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>c)</strong> How does the output of the network change if you use sigmoid in the hidden layers and ReLU in the output layer?</p>
|
||
</section>
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercise-5-processing-multiple-inputs-at-once">
|
||
<h1>Exercise 5 - Processing multiple inputs at once<a class="headerlink" href="#exercise-5-processing-multiple-inputs-at-once" title="Link to this heading">#</a></h1>
|
||
<p>So far, the feed forward function has taken one input vector as an input. This vector then undergoes a linear transformation and then an element-wise non-linear operation for each layer. This approach of sending one vector in at a time is great for interpreting how the network transforms data with its linear and non-linear operations, but not the best for numerical efficiency. Now, we want to be able to send many inputs through the network at once. This will make the code a bit harder to understand, but it will make it faster, and more compact. It will be worth the trouble.</p>
|
||
<p>To process multiple inputs at once, while still performing the same operations, you will only need to flip a couple things around.</p>
|
||
<p><strong>a)</strong> Complete the function <code class="docutils literal notranslate"><span class="pre">create_layers_batch</span></code> so that the weight matrix is the transpose of what it was when you only sent in one input at a time.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">create_layers_batch</span><span class="p">(</span><span class="n">network_input_size</span><span class="p">,</span> <span class="n">layer_output_sizes</span><span class="p">):</span>
|
||
<span class="n">layers</span> <span class="o">=</span> <span class="p">[]</span>
|
||
|
||
<span class="n">i_size</span> <span class="o">=</span> <span class="n">network_input_size</span>
|
||
<span class="k">for</span> <span class="n">layer_output_size</span> <span class="ow">in</span> <span class="n">layer_output_sizes</span><span class="p">:</span>
|
||
<span class="n">W</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">b</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">layers</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">))</span>
|
||
|
||
<span class="n">i_size</span> <span class="o">=</span> <span class="n">layer_output_size</span>
|
||
<span class="k">return</span> <span class="n">layers</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>b)</strong> Make a matrix of inputs with the shape (number of inputs, number of features), you choose the number of inputs and features per input. Then complete the function <code class="docutils literal notranslate"><span class="pre">feed_forward_batch</span></code> so that you can process this matrix of inputs with only one matrix multiplication and one broadcasted vector addition per layer. (Hint: You will only need to swap two variable around from your previous implementation, but remember to test that you get the same results for equivelent inputs!)</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">inputs</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">1000</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">feed_forward_batch</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">):</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="n">inputs</span>
|
||
<span class="k">for</span> <span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">),</span> <span class="n">activation_func</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">):</span>
|
||
<span class="n">z</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="k">return</span> <span class="n">a</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>c)</strong> Create and evaluate a neural network with 4 input features, and layers with output sizes 12, 10, 3 and activations ReLU, ReLU, softmax.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">network_input_size</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="n">layer_output_sizes</span> <span class="o">=</span> <span class="p">[</span><span class="o">...</span><span class="p">]</span>
|
||
<span class="n">activation_funcs</span> <span class="o">=</span> <span class="p">[</span><span class="o">...</span><span class="p">]</span>
|
||
<span class="n">layers</span> <span class="o">=</span> <span class="n">create_layers_batch</span><span class="p">(</span><span class="n">network_input_size</span><span class="p">,</span> <span class="n">layer_output_sizes</span><span class="p">)</span>
|
||
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">network_input_size</span><span class="p">)</span>
|
||
<span class="n">feed_forward_batch</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>You should use this batched approach moving forward, as it will lead to much more compact code. However, remember that each input is still treated separately, and that you will need to keep in mind the transposed weight matrix and other details when implementing backpropagation.</p>
|
||
</section>
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercise-6-predicting-on-real-data">
|
||
<h1>Exercise 6 - Predicting on real data<a class="headerlink" href="#exercise-6-predicting-on-real-data" title="Link to this heading">#</a></h1>
|
||
<p>You will now evaluate your neural network on the iris data set (<a class="reference external" href="https://scikit-learn.org/1.5/auto_examples/datasets/plot_iris_dataset.html">https://scikit-learn.org/1.5/auto_examples/datasets/plot_iris_dataset.html</a>).</p>
|
||
<p>This dataset contains data on 150 flowers of 3 different types which can be separated pretty well using the four features given for each flower, which includes the width and length of their leaves. You are will later train your network to actually make good predictions.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">iris</span> <span class="o">=</span> <span class="n">datasets</span><span class="o">.</span><span class="n">load_iris</span><span class="p">()</span>
|
||
|
||
<span class="n">_</span><span class="p">,</span> <span class="n">ax</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplots</span><span class="p">()</span>
|
||
<span class="n">scatter</span> <span class="o">=</span> <span class="n">ax</span><span class="o">.</span><span class="n">scatter</span><span class="p">(</span><span class="n">iris</span><span class="o">.</span><span class="n">data</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">],</span> <span class="n">iris</span><span class="o">.</span><span class="n">data</span><span class="p">[:,</span> <span class="mi">1</span><span class="p">],</span> <span class="n">c</span><span class="o">=</span><span class="n">iris</span><span class="o">.</span><span class="n">target</span><span class="p">)</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">xlabel</span><span class="o">=</span><span class="n">iris</span><span class="o">.</span><span class="n">feature_names</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">ylabel</span><span class="o">=</span><span class="n">iris</span><span class="o">.</span><span class="n">feature_names</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
||
<span class="n">_</span> <span class="o">=</span> <span class="n">ax</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span>
|
||
<span class="n">scatter</span><span class="o">.</span><span class="n">legend_elements</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="n">iris</span><span class="o">.</span><span class="n">target_names</span><span class="p">,</span> <span class="n">loc</span><span class="o">=</span><span class="s2">"lower right"</span><span class="p">,</span> <span class="n">title</span><span class="o">=</span><span class="s2">"Classes"</span>
|
||
<span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">inputs</span> <span class="o">=</span> <span class="n">iris</span><span class="o">.</span><span class="n">data</span>
|
||
|
||
<span class="c1"># Since each prediction is a vector with a score for each of the three types of flowers,</span>
|
||
<span class="c1"># we need to make each target a vector with a 1 for the correct flower and a 0 for the others.</span>
|
||
<span class="n">targets</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="n">iris</span><span class="o">.</span><span class="n">data</span><span class="p">),</span> <span class="mi">3</span><span class="p">))</span>
|
||
<span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">t</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">iris</span><span class="o">.</span><span class="n">target</span><span class="p">):</span>
|
||
<span class="n">targets</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="n">t</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">accuracy</span><span class="p">(</span><span class="n">predictions</span><span class="p">,</span> <span class="n">targets</span><span class="p">):</span>
|
||
<span class="n">one_hot_predictions</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">predictions</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">prediction</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">predictions</span><span class="p">):</span>
|
||
<span class="n">one_hot_predictions</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">prediction</span><span class="p">)]</span> <span class="o">=</span> <span class="mi">1</span>
|
||
<span class="k">return</span> <span class="n">accuracy_score</span><span class="p">(</span><span class="n">one_hot_predictions</span><span class="p">,</span> <span class="n">targets</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>a)</strong> What should the input size for the network be with this dataset? What should the output size of the last layer be?</p>
|
||
<p><strong>b)</strong> Create a network with two hidden layers, the first with sigmoid activation and the last with softmax, the first layer should have 8 “nodes”, the second has the number of nodes you found in exercise a). Softmax returns a “probability distribution”, in the sense that the numbers in the output are positive and add up to 1 and, their magnitude are in some sense relative to their magnitude before going through the softmax function. Remember to use the batched version of the create_layers and feed forward functions.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="o">...</span>
|
||
<span class="n">layers</span> <span class="o">=</span> <span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>c)</strong> Evaluate your model on the entire iris dataset! For later purposes, we will split the data into train and test sets, and compute gradients on smaller batches of the training data. But for now, evaluate the network on the whole thing at once.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">predictions</span> <span class="o">=</span> <span class="n">feed_forward_batch</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>d)</strong> Compute the accuracy of your model using the accuracy function defined above. Recreate your model a couple times and see how the accuracy changes.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="nb">print</span><span class="p">(</span><span class="n">accuracy</span><span class="p">(</span><span class="n">predictions</span><span class="p">,</span> <span class="n">targets</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercise-7-training-on-real-data-optional">
|
||
<h1>Exercise 7 - Training on real data (Optional)<a class="headerlink" href="#exercise-7-training-on-real-data-optional" title="Link to this heading">#</a></h1>
|
||
<p>To be able to actually do anything useful with your neural network, you need to train it. For this, we need a cost function and a way to take the gradient of the cost function wrt. the network parameters. The following exercises guide you through taking the gradient using autograd, and updating the network parameters using the gradient. Feel free to implement gradient methods like ADAM if you finish everything.</p>
|
||
<p>Since we are doing a classification task with multiple output classes, we use the cross-entropy loss function, which can evaluate performance on classification tasks. It sees if your prediction is “most certain” on the correct target.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">cross_entropy</span><span class="p">(</span><span class="n">predict</span><span class="p">,</span> <span class="n">target</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="o">-</span><span class="n">target</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="n">predict</span><span class="p">))</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">cost</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">,</span> <span class="n">target</span><span class="p">):</span>
|
||
<span class="n">predict</span> <span class="o">=</span> <span class="n">feed_forward_batch</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">cross_entropy</span><span class="p">(</span><span class="n">predict</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>To improve our network on whatever prediction task we have given it, we need to use a sensible cost function, take the gradient of that cost function with respect to our network parameters, the weights and biases, and then update the weights and biases using these gradients. To clarify, we need to find and use these</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C}{\partial W}, \frac{\partial C}{\partial b}
|
||
\]</div>
|
||
<p>Now we need to compute these gradients. This is pretty hard to do for a neural network, we will use most of next week to do this, but we can also use autograd to just do it for us, which is what we always do in practice. With the code cell below, we create a function which takes all of these gradients for us.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">autograd</span> <span class="kn">import</span> <span class="n">grad</span>
|
||
|
||
|
||
<span class="n">gradient_func</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span>
|
||
<span class="n">cost</span><span class="p">,</span> <span class="mi">1</span>
|
||
<span class="p">)</span> <span class="c1"># Taking the gradient wrt. the second input to the cost function, i.e. the layers</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>a)</strong> What shape should the gradient of the cost function wrt. weights and biases be?</p>
|
||
<p><strong>b)</strong> Use the <code class="docutils literal notranslate"><span class="pre">gradient_func</span></code> function to take the gradient of the cross entropy wrt. the weights and biases of the network. Check the shapes of what’s inside. What does the <code class="docutils literal notranslate"><span class="pre">grad</span></code> func from autograd actually do?</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">layers_grad</span> <span class="o">=</span> <span class="n">gradient_func</span><span class="p">(</span>
|
||
<span class="n">inputs</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">,</span> <span class="n">targets</span>
|
||
<span class="p">)</span> <span class="c1"># Don't change this</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>c)</strong> Finish the <code class="docutils literal notranslate"><span class="pre">train_network</span></code> function.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">train_network</span><span class="p">(</span>
|
||
<span class="n">inputs</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">,</span> <span class="n">targets</span><span class="p">,</span> <span class="n">learning_rate</span><span class="o">=</span><span class="mf">0.001</span><span class="p">,</span> <span class="n">epochs</span><span class="o">=</span><span class="mi">100</span>
|
||
<span class="p">):</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">epochs</span><span class="p">):</span>
|
||
<span class="n">layers_grad</span> <span class="o">=</span> <span class="n">gradient_func</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">activation_funcs</span><span class="p">,</span> <span class="n">targets</span><span class="p">)</span>
|
||
<span class="k">for</span> <span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">b</span><span class="p">),</span> <span class="p">(</span><span class="n">W_g</span><span class="p">,</span> <span class="n">b_g</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">layers</span><span class="p">,</span> <span class="n">layers_grad</span><span class="p">):</span>
|
||
<span class="n">W</span> <span class="o">-=</span> <span class="o">...</span>
|
||
<span class="n">b</span> <span class="o">-=</span> <span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>e)</strong> What do we call the gradient method used above?</p>
|
||
<p><strong>d)</strong> Train your network and see how the accuracy changes! Make a plot if you want.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>e)</strong> How high of an accuracy is it possible to acheive with a neural network on this dataset, if we use the whole thing as training data?</p>
|
||
</section>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
name: "python3",
|
||
path: "./."
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</article>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<footer class="prev-next-footer d-print-none">
|
||
|
||
<div class="prev-next-area">
|
||
<a class="left-prev"
|
||
href="week41.html"
|
||
title="previous page">
|
||
<i class="fa-solid fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title">Week 41 Neural networks and constructing a neural network code</p>
|
||
</div>
|
||
</a>
|
||
<a class="right-next"
|
||
href="week42.html"
|
||
title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title">Week 42 Constructing a Neural Network code with examples</p>
|
||
</div>
|
||
<i class="fa-solid fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
|
||
|
||
|
||
<div class="sidebar-secondary-item">
|
||
<div class="page-toc tocsection onthispage">
|
||
<i class="fa-solid fa-list"></i> Contents
|
||
</div>
|
||
<nav class="bd-toc-nav page-toc">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#">Exercises week 41</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#overarching-aims-of-the-exercises-this-week">Overarching aims of the exercises this week</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-1">Exercise 1</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-2">Exercise 2</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-3">Exercise 3</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-4-custom-activation-for-each-layer">Exercise 4 - Custom activation for each layer</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-5-processing-multiple-inputs-at-once">Exercise 5 - Processing multiple inputs at once</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-6-predicting-on-real-data">Exercise 6 - Predicting on real data</a></li>
|
||
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-7-training-on-real-data-optional">Exercise 7 - Training on real data (Optional)</a></li>
|
||
</ul>
|
||
|
||
</nav></div>
|
||
|
||
</div></div>
|
||
|
||
|
||
</div>
|
||
<footer class="bd-footer-content">
|
||
|
||
<div class="bd-footer-content__inner container">
|
||
|
||
<div class="footer-item">
|
||
|
||
<p class="component-author">
|
||
By Morten Hjorth-Jensen
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
|
||
<p class="copyright">
|
||
|
||
© Copyright 2023.
|
||
<br/>
|
||
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</footer>
|
||
|
||
|
||
</main>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
||
<script src="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
<script src="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<footer class="bd-footer">
|
||
</footer>
|
||
</body>
|
||
</html> |