Morten Hjorth-Jensen
930 lines
68 KiB
HTML
930 lines
68 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
<html>
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<title>12. Clustering and Unsupervised Learning — Applied Data Analysis and Machine Learning</title>
|
||
|
||
<link href="_static/css/theme.css" rel="stylesheet">
|
||
<link href="_static/css/index.ff1ffe594081f20da1ef19478df9384b.css" rel="stylesheet">
|
||
|
||
|
||
<link rel="stylesheet"
|
||
href="_static/vendor/fontawesome/5.13.0/css/all.min.css">
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin
|
||
href="_static/vendor/fontawesome/5.13.0/webfonts/fa-solid-900.woff2">
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin
|
||
href="_static/vendor/fontawesome/5.13.0/webfonts/fa-brands-400.woff2">
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-book-theme.css?digest=c3fdc42140077d1ad13ad2f1588a4309" />
|
||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/mystnb.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/panels-main.c949a650a448cc0ae9fd3441c0e17fb0.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/panels-variables.06eb56fa6e07937060861dad626602ad.css" />
|
||
|
||
<link rel="preload" as="script" href="_static/js/index.be7d3bbb2ef33a8344ce.js">
|
||
|
||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||
<script src="_static/jquery.js"></script>
|
||
<script src="_static/underscore.js"></script>
|
||
<script src="_static/doctools.js"></script>
|
||
<script src="_static/clipboard.min.js"></script>
|
||
<script src="_static/copybutton.js"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="_static/togglebutton.js"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown, .tag_hide_input div.cell_input, .tag_hide-input div.cell_input, .tag_hide_output div.cell_output, .tag_hide-output div.cell_output, .tag_hide_cell.cell, .tag_hide-cell.cell';</script>
|
||
<script src="_static/sphinx-book-theme.d59cb220de22ca1c485ebbdc042f0030.js"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"
|
||
const thebe_selector = ".thebe,.cell"
|
||
const thebe_selector_input = "pre"
|
||
const thebe_selector_output = ".output, .cell_output"
|
||
</script>
|
||
<script async="async" src="_static/sphinx-thebe.js"></script>
|
||
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
||
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="13. Neural networks" href="chapter9.html" />
|
||
<link rel="prev" title="11. Basic ideas of the Principal Component Analysis (PCA)" href="chapter8.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
<meta name="docsearch:language" content="None">
|
||
|
||
|
||
<!-- Google Analytics -->
|
||
|
||
</head>
|
||
<body data-spy="scroll" data-target="#bd-toc-nav" data-offset="80">
|
||
|
||
<div class="container-fluid" id="banner"></div>
|
||
|
||
|
||
|
||
<div class="container-xl">
|
||
<div class="row">
|
||
|
||
<div class="col-12 col-md-3 bd-sidebar site-navigation show" id="site-navigation">
|
||
|
||
<div class="navbar-brand-box">
|
||
<a class="navbar-brand text-wrap" href="index.html">
|
||
|
||
<!-- `logo` is deprecated in Sphinx 4.0, so remove this when we stop supporting 3 -->
|
||
|
||
|
||
|
||
<img src="_static/logo.png" class="logo" alt="logo">
|
||
|
||
|
||
<h1 class="site-logo" id="site-title">Applied Data Analysis and Machine Learning</h1>
|
||
|
||
</a>
|
||
</div><form class="bd-search d-flex align-items-center" action="search.html" method="get">
|
||
<i class="icon fas fa-search"></i>
|
||
<input type="search" class="form-control" name="q" id="search-input" placeholder="Search this book..." aria-label="Search this book..." autocomplete="off" >
|
||
</form><nav class="bd-links" id="bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item active">
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="intro.html">
|
||
Applied Data Analysis and Machine Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
About the course
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="schedule.html">
|
||
Teaching schedule with links to material
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="teachers.html">
|
||
Teachers and Grading
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="textbooks.html">
|
||
Textbooks
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Review of Statistics with Resampling Techniques and Linear Algebra
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="statistics.html">
|
||
1. Elements of Probability Theory and Statistical Data Analysis
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="linalg.html">
|
||
2. Linear Algebra, Handling of Arrays and more Python Features
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
From Regression to Support Vector Machines
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter1.html">
|
||
3. Linear Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter2.html">
|
||
4. Ridge and Lasso Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter3.html">
|
||
5. Resampling Methods
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter4.html">
|
||
6. Logistic Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapteroptimization.html">
|
||
7. Optimization, the central part of any Machine Learning algortithm
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter5.html">
|
||
8. Support Vector Machines, overarching aims
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Decision Trees, Ensemble Methods and Boosting
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter6.html">
|
||
9. Decision trees, overarching aims
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter7.html">
|
||
10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Dimensionality Reduction
|
||
</span>
|
||
</p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter8.html">
|
||
11. Basic ideas of the Principal Component Analysis (PCA)
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1 current active">
|
||
<a class="current reference internal" href="#">
|
||
12. Clustering and Unsupervised Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Deep Learning Methods
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter9.html">
|
||
13. Neural networks
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter10.html">
|
||
14. Building a Feed Forward Neural Network
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter11.html">
|
||
15. Solving Differential Equations with Deep Learning
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter12.html">
|
||
16. Convolutional Neural Networks
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter13.html">
|
||
17. Recurrent neural networks: Overarching view
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Weekly material, notes and exercises
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek34.html">
|
||
Exercises week 34
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week34.html">
|
||
Week 34: Introduction to the course, Logistics and Practicalities
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek35.html">
|
||
Exercises week 35
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week35.html">
|
||
Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek36.html">
|
||
Exercises week 36
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week36.html">
|
||
Week 36: Linear Regression and Statistical interpretations
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek37.html">
|
||
Exercises week 37
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week37.html">
|
||
Week 37: Statistical interpretations and Resampling Methods
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek38.html">
|
||
Exercises week 38
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week38.html">
|
||
Week 38: Logistic Regression and Optimization
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Projects
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="project1.html">
|
||
Project 1 on Machine Learning, deadline October 7 (midnight), 2024
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</div>
|
||
</nav> <!-- To handle the deprecated key -->
|
||
|
||
<div class="navbar_extra_footer">
|
||
Powered by <a href="https://jupyterbook.org">Jupyter Book</a>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<main class="col py-md-3 pl-md-4 bd-content overflow-auto" role="main">
|
||
|
||
<div class="topbar container-xl fixed-top">
|
||
<div class="topbar-contents row">
|
||
<div class="col-12 col-md-3 bd-topbar-whitespace site-navigation show"></div>
|
||
<div class="col pl-md-4 topbar-main">
|
||
|
||
<button id="navbar-toggler" class="navbar-toggler ml-0" type="button" data-toggle="collapse"
|
||
data-toggle="tooltip" data-placement="bottom" data-target=".site-navigation" aria-controls="navbar-menu"
|
||
aria-expanded="true" aria-label="Toggle navigation" aria-controls="site-navigation"
|
||
title="Toggle navigation" data-toggle="tooltip" data-placement="left">
|
||
<i class="fas fa-bars"></i>
|
||
<i class="fas fa-arrow-left"></i>
|
||
<i class="fas fa-arrow-up"></i>
|
||
</button>
|
||
|
||
|
||
<div class="dropdown-buttons-trigger">
|
||
<button id="dropdown-buttons-trigger" class="btn btn-secondary topbarbtn" aria-label="Download this page"><i
|
||
class="fas fa-download"></i></button>
|
||
|
||
<div class="dropdown-buttons">
|
||
<!-- ipynb file if we had a myst markdown file -->
|
||
|
||
<!-- Download raw file -->
|
||
<a class="dropdown-buttons" href="_sources/clustering.ipynb"><button type="button"
|
||
class="btn btn-secondary topbarbtn" title="Download source file" data-toggle="tooltip"
|
||
data-placement="left">.ipynb</button></a>
|
||
<!-- Download PDF via print -->
|
||
<button type="button" id="download-print" class="btn btn-secondary topbarbtn" title="Print to PDF"
|
||
onclick="printPdf(this)" data-toggle="tooltip" data-placement="left">.pdf</button>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Source interaction buttons -->
|
||
|
||
<!-- Full screen (wrap in <a> to have style consistency -->
|
||
|
||
<a class="full-screen-button"><button type="button" class="btn btn-secondary topbarbtn" data-toggle="tooltip"
|
||
data-placement="bottom" onclick="toggleFullScreen()" aria-label="Fullscreen mode"
|
||
title="Fullscreen mode"><i
|
||
class="fas fa-expand"></i></button></a>
|
||
|
||
<!-- Launch buttons -->
|
||
|
||
</div>
|
||
|
||
<!-- Table of contents -->
|
||
<div class="d-none d-md-block col-md-2 bd-toc show noprint">
|
||
|
||
<div class="tocsection onthispage pt-5 pb-3">
|
||
<i class="fas fa-list"></i> Contents
|
||
</div>
|
||
<nav id="bd-toc-nav" aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#codes-and-approaches">
|
||
12.1. Codes and Approaches
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div id="main-content" class="row">
|
||
<div class="col-12 col-md-9 pl-md-3 pr-md-0">
|
||
<!-- Table of contents that is only displayed when printing the page -->
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>Clustering and Unsupervised Learning</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#codes-and-approaches">
|
||
12.1. Codes and Approaches
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div>
|
||
|
||
<!-- HTML file automatically generated from DocOnce source (https://github.com/doconce/doconce/)
|
||
doconce format html clustering.do.txt --><div class="tex2jax_ignore mathjax_ignore section" id="clustering-and-unsupervised-learning">
|
||
<h1><span class="section-number">12. </span>Clustering and Unsupervised Learning<a class="headerlink" href="#clustering-and-unsupervised-learning" title="Permalink to this headline">¶</a></h1>
|
||
<p>In general terms cluster analysis, or clustering, is the task of grouping a
|
||
data-set into different distinct categories based on some measure of equality of
|
||
the data. This measure is often referred to as a <strong>metric</strong> or <strong>similarity
|
||
measure</strong> in the literature (note: sometimes we deal with a <strong>dissimilarity
|
||
measure</strong> instead). Usually, these metrics are formulated as some kind of
|
||
distance function between points in a high-dimensional space.</p>
|
||
<p>The simplest, and also the most
|
||
common is the <strong>Euclidean distance</strong>.</p>
|
||
<p>The simplest of all clustering algorithms is the <strong>k-means algorithm</strong>
|
||
, sometimes also referred to as <em>Lloyds algorithm</em>. It is the simplest and also
|
||
the most common. From its simplicity it obtains both strengths and weaknesses.
|
||
These will be discussed in more detail later. The <span class="math notranslate nohighlight">\(k\)</span>-means algorithm is a
|
||
<strong>centroid based</strong> clustering algorithm.</p>
|
||
<p>Assume, we are given <span class="math notranslate nohighlight">\(n\)</span> data points and we wish to split the data into <span class="math notranslate nohighlight">\(K < n\)</span>
|
||
different categories, or clusters. We label each cluster by an integer</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
k\in\{1, \cdots, K \}.
|
||
\]</div>
|
||
<p>In the basic k-means algorithm each point is assigned to only
|
||
one cluster <span class="math notranslate nohighlight">\(k\)</span>, and these assignments are <em>non-injective</em> i.e. many-to-one. We
|
||
can think of these mappings as an encoder <span class="math notranslate nohighlight">\(k = C(i)\)</span>, which assigns the <span class="math notranslate nohighlight">\(i\)</span>-th
|
||
data-point <span class="math notranslate nohighlight">\(\bf x_i\)</span> to the <span class="math notranslate nohighlight">\(k\)</span>-th cluster.</p>
|
||
<p><span class="math notranslate nohighlight">\(k\)</span>-means algorithm in words:</p>
|
||
<ol class="simple">
|
||
<li><p>We start with guesses / random initializations of our <span class="math notranslate nohighlight">\(k\)</span> cluster centers/centroids</p></li>
|
||
<li><p>For each centroid the points that are most similar are identified</p></li>
|
||
<li><p>Then we move / replace each centroid with a coordinate average of all the points that were assigned to that centroid.</p></li>
|
||
<li><p>Iterate 2-3 until the centroids no longer move (to some tolerance)</p></li>
|
||
</ol>
|
||
<p>We assume we have <span class="math notranslate nohighlight">\(n\)</span> data-points</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="eq:kmeanspoints"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}\label{eq:kmeanspoints} \tag{1}
|
||
\boldsymbol{x_i} = \{x_{i, 1}, \cdots, x_{i, p}\}\in\mathbb{R}^p.
|
||
\end{equation}
|
||
\]</div>
|
||
<p>which we wish to group into <span class="math notranslate nohighlight">\(K < n\)</span> clusters. For our dissimilarity measure we
|
||
use the <em>squared Euclidean distance</em></p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="eq:squaredeuclidean"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}\label{eq:squaredeuclidean} \tag{2}
|
||
d(\boldsymbol{x_i}, \boldsymbol{x_i'}) = \sum_{j=1}^p(x_{ij} - x_{i'j})^2
|
||
= ||\boldsymbol{x_i} - \boldsymbol{x_{i'}}||^2
|
||
\end{equation}
|
||
\]</div>
|
||
<p>We define the so called <em>within-cluster point scatter</em> which gives us a
|
||
measure of how close each data point assigned to the same cluster tends to be to
|
||
the all the others.</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="eq:withincluster"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}\label{eq:withincluster} \tag{3}
|
||
W(C) = \frac{1}{2}\sum_{k=1}^K\sum_{C(i)=k}
|
||
\sum_{C(i')=k}d(\boldsymbol{x_i}, \boldsymbol{x_{i'}}) =
|
||
\sum_{k=1}^KN_k\sum_{C(i)=k}||\boldsymbol{x_i} - \boldsymbol{\overline{x_k}}||^2
|
||
\end{equation}
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\boldsymbol{\overline{x_k}}\)</span> is the mean vector associated with the <span class="math notranslate nohighlight">\(k\)</span>-th
|
||
cluster, and <span class="math notranslate nohighlight">\(N_k = \sum_{i=1}^nI(C(i) = k)\)</span>, where the <span class="math notranslate nohighlight">\(I()\)</span> notation is
|
||
similar to the Kronecker delta (<em>Commonly used in statistics, it just means that
|
||
when <span class="math notranslate nohighlight">\(i = k\)</span> we have the encoder <span class="math notranslate nohighlight">\(C(i)\)</span></em>). In other words, the within-cluster
|
||
scatter measures the compactness of each cluster with respect to the data points
|
||
assigned to each cluster. This is the quantity that the <span class="math notranslate nohighlight">\(k\)</span>-means algorithm aims
|
||
to minimize. We refer to this quantity <span class="math notranslate nohighlight">\(W(C)\)</span> as the within cluster scatter
|
||
because of its relation to the <em>total scatter</em>.</p>
|
||
<p>We have</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="eq:totalscatter"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}\label{eq:totalscatter} \tag{4}
|
||
T = W(C) + B(C) = \frac{1}{2}\sum_{i=1}^n
|
||
\sum_{i'=1}^nd(\boldsymbol{x_i}, \boldsymbol{x_{i'}})
|
||
= \frac{1}{2}\sum_{k=1}^K\sum_{C(i)=k}
|
||
\Big(\sum_{C(i') = k}d(\boldsymbol{x_i}, \boldsymbol{x_{i'}})
|
||
+ \sum_{C(i')\neq k}d(\boldsymbol{x_i}, \boldsymbol{x_{i'}})\Big).
|
||
\end{equation}
|
||
\]</div>
|
||
<p>This is a quantity that is conserved throughout the <span class="math notranslate nohighlight">\(k\)</span>-means algorithm. It can
|
||
be thought of as the total amount of information in the data, and it is composed
|
||
of the aforementioned within-cluster scatter and the <em>between-cluster scatter</em>
|
||
<span class="math notranslate nohighlight">\(B(C)\)</span>. In methods such as principle component analysis the total scatter is not
|
||
conserved.</p>
|
||
<p>Given a cluster mean <span class="math notranslate nohighlight">\(\boldsymbol{m_k}\)</span> we define the <strong>total cluster variance</strong></p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="eq:totalclustervariance"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}\label{eq:totalclustervariance} \tag{5}
|
||
\min_{C, \{\boldsymbol{m_k}\}_1^K}\sum_{k=1}^KN_k\sum||\boldsymbol{x_i} - \boldsymbol{m_k}||^2
|
||
\end{equation}
|
||
\]</div>
|
||
<p>Now we have all the pieces necessary to formally revisit the <span class="math notranslate nohighlight">\(k\)</span>-means algorithm.</p>
|
||
<p>The <span class="math notranslate nohighlight">\(k\)</span>-means clustering algorithm goes as follows</p>
|
||
<ol class="simple">
|
||
<li><p>For a given cluster assignment <span class="math notranslate nohighlight">\(C\)</span>, and <span class="math notranslate nohighlight">\(k\)</span> cluster means <span class="math notranslate nohighlight">\(\left\{m_1, \cdots, m_k\right\}\)</span>. We minimize the total cluster variance with respect to the cluster means <span class="math notranslate nohighlight">\(\{m_k\}\)</span> yielding the means of the currently assigned clusters.</p></li>
|
||
<li><p>Given a current set of <span class="math notranslate nohighlight">\(k\)</span> means <span class="math notranslate nohighlight">\(\{m_k\}\)</span> the total cluster variance is minimized by assigning each observation to the closest (current) cluster mean. That is $<span class="math notranslate nohighlight">\(C(i) = \underset{1\leq k\leq K}{\mathrm{argmin}} ||\boldsymbol{x_i} - \boldsymbol{m_k}||^2\)</span>$</p></li>
|
||
<li><p>Steps 1 and 2 are repeated until the assignments do not change.</p></li>
|
||
</ol>
|
||
<div class="section" id="codes-and-approaches">
|
||
<h2><span class="section-number">12.1. </span>Codes and Approaches<a class="headerlink" href="#codes-and-approaches" title="Permalink to this headline">¶</a></h2>
|
||
<ol class="simple">
|
||
<li><p>Before we start we specify a number <span class="math notranslate nohighlight">\(k\)</span> which is the number of clusters we want to try to separate our data into.</p></li>
|
||
<li><p>We initially choose <span class="math notranslate nohighlight">\(k\)</span> random data points in our data as our initial centroids, <em>or means</em> (this is where the name comes from).</p></li>
|
||
<li><p>Assign each data point to their closest centroid, based on the squared Euclidean distance.</p></li>
|
||
<li><p>For each of the <span class="math notranslate nohighlight">\(k\)</span> cluster we update the centroid by calculating new mean values for all the data points in the cluster.</p></li>
|
||
<li><p>Iteratively minimize the within cluster scatter by performing steps (3, 4) until the new assignments stop changing (can be to some tolerance) or until a maximum number of iterations have passed.</p></li>
|
||
</ol>
|
||
<p>Let us now program the most basic version of the algorithm using nothing but
|
||
Python with numpy arrays. This code is kept intentionally simple to gradually
|
||
progress our understanding. There is no vectorization of any kind, and even most
|
||
helper functions are not utilized.</p>
|
||
<p>We need first a dataset to do our cluster analysis on. In our case
|
||
this is a plain <em>vanilla</em> data set using random numbers using a
|
||
Gaussian distribution.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="o">%</span><span class="k">matplotlib</span> inline
|
||
|
||
<span class="kn">import</span> <span class="nn">time</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">tensorflow</span> <span class="k">as</span> <span class="nn">tf</span>
|
||
<span class="kn">from</span> <span class="nn">matplotlib</span> <span class="kn">import</span> <span class="n">image</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.cluster</span> <span class="kn">import</span> <span class="n">KMeans</span>
|
||
<span class="kn">from</span> <span class="nn">IPython.display</span> <span class="kn">import</span> <span class="n">display</span>
|
||
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">2021</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output traceback highlight-ipythontb notranslate"><div class="highlight"><pre><span></span><span class="gt">---------------------------------------------------------------------------</span>
|
||
<span class="ne">NotFoundError</span><span class="g g-Whitespace"> </span>Traceback (most recent call last)
|
||
<span class="n">Cell</span> <span class="n">In</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">line</span> <span class="mi">5</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">3</span> <span class="kn">import</span> <span class="nn">time</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">4</span> <span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="ne">----> </span><span class="mi">5</span> <span class="kn">import</span> <span class="nn">tensorflow</span> <span class="k">as</span> <span class="nn">tf</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">6</span> <span class="kn">from</span> <span class="nn">matplotlib</span> <span class="kn">import</span> <span class="n">image</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">7</span> <span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
|
||
<span class="n">File</span> <span class="o">~/</span><span class="n">miniforge3</span><span class="o">/</span><span class="n">envs</span><span class="o">/</span><span class="n">myenv</span><span class="o">/</span><span class="n">lib</span><span class="o">/</span><span class="n">python3</span><span class="mf">.9</span><span class="o">/</span><span class="n">site</span><span class="o">-</span><span class="n">packages</span><span class="o">/</span><span class="n">tensorflow</span><span class="o">/</span><span class="fm">__init__</span><span class="o">.</span><span class="n">py</span><span class="p">:</span><span class="mi">443</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">441</span> <span class="n">_plugin_dir</span> <span class="o">=</span> <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">_s</span><span class="p">,</span> <span class="s1">'tensorflow-plugins'</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">442</span> <span class="k">if</span> <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">_plugin_dir</span><span class="p">):</span>
|
||
<span class="ne">--> </span><span class="mi">443</span> <span class="n">_ll</span><span class="o">.</span><span class="n">load_library</span><span class="p">(</span><span class="n">_plugin_dir</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">444</span> <span class="c1"># Load Pluggable Device Library</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">445</span> <span class="n">_ll</span><span class="o">.</span><span class="n">load_pluggable_device_library</span><span class="p">(</span><span class="n">_plugin_dir</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/tensorflow/python/framework/load_library.py:151,</span> in <span class="ni">load_library</span><span class="nt">(library_location)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">148</span> <span class="n">kernel_libraries</span> <span class="o">=</span> <span class="p">[</span><span class="n">library_location</span><span class="p">]</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">150</span> <span class="k">for</span> <span class="n">lib</span> <span class="ow">in</span> <span class="n">kernel_libraries</span><span class="p">:</span>
|
||
<span class="ne">--> </span><span class="mi">151</span> <span class="n">py_tf</span><span class="o">.</span><span class="n">TF_LoadLibrary</span><span class="p">(</span><span class="n">lib</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">153</span> <span class="k">else</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">154</span> <span class="k">raise</span> <span class="ne">OSError</span><span class="p">(</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">155</span> <span class="n">errno</span><span class="o">.</span><span class="n">ENOENT</span><span class="p">,</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">156</span> <span class="s1">'The file or folder to load kernel libraries from does not exist.'</span><span class="p">,</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">157</span> <span class="n">library_location</span><span class="p">)</span>
|
||
|
||
<span class="ne">NotFoundError</span>: dlopen(/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/tensorflow-plugins/libmetal_plugin.dylib, 0x0006): symbol not found in flat namespace '_TF_GetInputPropertiesList'
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Next we define functions, for ease of use later, to generate Gaussians and to
|
||
set up our toy data set.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">gaussian_points</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">n_points</span><span class="o">=</span><span class="mi">1000</span><span class="p">,</span> <span class="n">mean_vector</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">]),</span>
|
||
<span class="n">sample_variance</span><span class="o">=</span><span class="mi">1</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Very simple custom function to generate gaussian distributed point clusters</span>
|
||
<span class="sd"> with variable dimension, number of points, means in each direction</span>
|
||
<span class="sd"> (must match dim) and sample variance.</span>
|
||
|
||
<span class="sd"> Inputs:</span>
|
||
<span class="sd"> dim (int)</span>
|
||
<span class="sd"> n_points (int)</span>
|
||
<span class="sd"> mean_vector (np.array) (where index 0 is x, index 1 is y etc.)</span>
|
||
<span class="sd"> sample_variance (float)</span>
|
||
|
||
<span class="sd"> Returns:</span>
|
||
<span class="sd"> data (np.array): with dimensions (dim x n_points)</span>
|
||
<span class="sd"> """</span>
|
||
|
||
<span class="n">mean_matrix</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">dim</span><span class="p">)</span> <span class="o">+</span> <span class="n">mean_vector</span>
|
||
<span class="n">covariance_matrix</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">eye</span><span class="p">(</span><span class="n">dim</span><span class="p">)</span> <span class="o">*</span> <span class="n">sample_variance</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">multivariate_normal</span><span class="p">(</span><span class="n">mean_matrix</span><span class="p">,</span> <span class="n">covariance_matrix</span><span class="p">,</span>
|
||
<span class="n">n_points</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">data</span>
|
||
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">generate_simple_clustering_dataset</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">n_points</span><span class="o">=</span><span class="mi">1000</span><span class="p">,</span> <span class="n">plotting</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||
<span class="n">return_data</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Toy model to illustrate k-means clustering</span>
|
||
<span class="sd"> """</span>
|
||
|
||
<span class="n">data1</span> <span class="o">=</span> <span class="n">gaussian_points</span><span class="p">(</span><span class="n">mean_vector</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">]))</span>
|
||
<span class="n">data2</span> <span class="o">=</span> <span class="n">gaussian_points</span><span class="p">()</span>
|
||
<span class="n">data3</span> <span class="o">=</span> <span class="n">gaussian_points</span><span class="p">(</span><span class="n">mean_vector</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mf">4.5</span><span class="p">]))</span>
|
||
<span class="n">data4</span> <span class="o">=</span> <span class="n">gaussian_points</span><span class="p">(</span><span class="n">mean_vector</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">]))</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">((</span><span class="n">data1</span><span class="p">,</span> <span class="n">data2</span><span class="p">,</span> <span class="n">data3</span><span class="p">,</span> <span class="n">data4</span><span class="p">),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="n">plotting</span><span class="p">:</span>
|
||
<span class="n">fig</span><span class="p">,</span> <span class="n">ax</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplots</span><span class="p">()</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">scatter</span><span class="p">(</span><span class="n">data</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">],</span> <span class="n">data</span><span class="p">[:,</span> <span class="mi">1</span><span class="p">],</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.2</span><span class="p">)</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s1">'Toy Model Dataset'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
|
||
|
||
<span class="k">if</span> <span class="n">return_data</span><span class="p">:</span>
|
||
<span class="k">return</span> <span class="n">data</span>
|
||
|
||
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">generate_simple_clustering_dataset</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>With the above dataset we start
|
||
implementing the <span class="math notranslate nohighlight">\(k\)</span>-means algorithm.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">n_samples</span><span class="p">,</span> <span class="n">dimensions</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">shape</span>
|
||
<span class="n">n_clusters</span> <span class="o">=</span> <span class="mi">4</span>
|
||
|
||
<span class="c1"># we randomly initialize our centroids</span>
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">2021</span><span class="p">)</span>
|
||
<span class="n">centroids</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">n_samples</span><span class="p">,</span> <span class="n">n_clusters</span><span class="p">,</span> <span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span> <span class="p">:]</span>
|
||
<span class="n">distances</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="n">n_samples</span><span class="p">,</span> <span class="n">n_clusters</span><span class="p">))</span>
|
||
|
||
<span class="c1"># first we need to calculate the distance to each centroid from our data</span>
|
||
<span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_clusters</span><span class="p">):</span>
|
||
<span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_samples</span><span class="p">):</span>
|
||
<span class="n">dist</span> <span class="o">=</span> <span class="mi">0</span>
|
||
<span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">dimensions</span><span class="p">):</span>
|
||
<span class="n">dist</span> <span class="o">+=</span> <span class="n">np</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span><span class="n">data</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="n">d</span><span class="p">]</span> <span class="o">-</span> <span class="n">centroids</span><span class="p">[</span><span class="n">k</span><span class="p">,</span> <span class="n">d</span><span class="p">])</span><span class="o">**</span><span class="mi">2</span>
|
||
<span class="n">distances</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">dist</span>
|
||
|
||
<span class="c1"># we initialize an array to keep track of to which cluster each point belongs</span>
|
||
<span class="c1"># the way we set it up here the index tracks which point and the value which</span>
|
||
<span class="c1"># cluster the point belongs to</span>
|
||
<span class="n">cluster_labels</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">n_samples</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="s1">'int'</span><span class="p">)</span>
|
||
|
||
<span class="c1"># next we loop through our samples and for every point assign it to the cluster</span>
|
||
<span class="c1"># to which it has the smallest distance to</span>
|
||
<span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_samples</span><span class="p">):</span>
|
||
<span class="c1"># tracking variables (all of this is basically just an argmin)</span>
|
||
<span class="n">smallest</span> <span class="o">=</span> <span class="mf">1e10</span>
|
||
<span class="n">smallest_row_index</span> <span class="o">=</span> <span class="mf">1e10</span>
|
||
<span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_clusters</span><span class="p">):</span>
|
||
<span class="k">if</span> <span class="n">distances</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">]</span> <span class="o"><</span> <span class="n">smallest</span><span class="p">:</span>
|
||
<span class="n">smallest</span> <span class="o">=</span> <span class="n">distances</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">]</span>
|
||
<span class="n">smallest_row_index</span> <span class="o">=</span> <span class="n">k</span>
|
||
|
||
<span class="n">cluster_labels</span><span class="p">[</span><span class="n">n</span><span class="p">]</span> <span class="o">=</span> <span class="n">smallest_row_index</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">fig</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">()</span>
|
||
<span class="n">ax</span> <span class="o">=</span> <span class="n">fig</span><span class="o">.</span><span class="n">add_subplot</span><span class="p">()</span>
|
||
<span class="n">unique_cluster_labels</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">unique</span><span class="p">(</span><span class="n">cluster_labels</span><span class="p">)</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">unique_cluster_labels</span><span class="p">:</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">scatter</span><span class="p">(</span><span class="n">data</span><span class="p">[</span><span class="n">cluster_labels</span> <span class="o">==</span> <span class="n">i</span><span class="p">,</span> <span class="mi">0</span><span class="p">],</span>
|
||
<span class="n">data</span><span class="p">[</span><span class="n">cluster_labels</span> <span class="o">==</span> <span class="n">i</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span>
|
||
<span class="n">label</span> <span class="o">=</span> <span class="n">i</span><span class="p">,</span>
|
||
<span class="n">alpha</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">)</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">scatter</span><span class="p">(</span><span class="n">centroids</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">],</span> <span class="n">centroids</span><span class="p">[:,</span> <span class="mi">1</span><span class="p">],</span> <span class="n">c</span><span class="o">=</span><span class="s1">'black'</span><span class="p">)</span>
|
||
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s2">"First Grouping of Points to Centroids"</span><span class="p">)</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>So what do we have so far? We have ‘picked’ <span class="math notranslate nohighlight">\(k\)</span> centroids at random from our
|
||
data points. There are other ways of more intelligently choosing their
|
||
initializations, however for our purposes randomly is fine. Then we have
|
||
initialized an array ‘distances’ which holds the information of the distance,
|
||
<em>or dissimilarity</em>, of every point to of our centroids. Finally, we have
|
||
initialized an array ‘cluster_labels’ which according to our distances array
|
||
holds the information of to which centroid every point is assigned. This was the
|
||
first pass of our algorithm. Essentially, all we need to do now is repeat the
|
||
distance and assignment steps above until we have reached a desired convergence
|
||
or a maximum amount of iterations.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">max_iterations</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">tolerance</span> <span class="o">=</span> <span class="mf">1e-8</span>
|
||
|
||
<span class="k">for</span> <span class="n">iteration</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">max_iterations</span><span class="p">):</span>
|
||
<span class="n">prev_centroids</span> <span class="o">=</span> <span class="n">centroids</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||
<span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_clusters</span><span class="p">):</span>
|
||
<span class="c1"># this array will be used to update our centroid positions</span>
|
||
<span class="n">vector_mean</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">dimensions</span><span class="p">)</span>
|
||
<span class="n">mean_divisor</span> <span class="o">=</span> <span class="mi">0</span>
|
||
<span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_samples</span><span class="p">):</span>
|
||
<span class="k">if</span> <span class="n">cluster_labels</span><span class="p">[</span><span class="n">n</span><span class="p">]</span> <span class="o">==</span> <span class="n">k</span><span class="p">:</span>
|
||
<span class="n">vector_mean</span> <span class="o">+=</span> <span class="n">data</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="p">:]</span>
|
||
<span class="n">mean_divisor</span> <span class="o">+=</span> <span class="mi">1</span>
|
||
|
||
<span class="c1"># update according to the k means</span>
|
||
<span class="n">centroids</span><span class="p">[</span><span class="n">k</span><span class="p">,</span> <span class="p">:]</span> <span class="o">=</span> <span class="n">vector_mean</span> <span class="o">/</span> <span class="n">mean_divisor</span>
|
||
|
||
<span class="c1"># we find the dissimilarity</span>
|
||
<span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_clusters</span><span class="p">):</span>
|
||
<span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_samples</span><span class="p">):</span>
|
||
<span class="n">dist</span> <span class="o">=</span> <span class="mi">0</span>
|
||
<span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">dimensions</span><span class="p">):</span>
|
||
<span class="n">dist</span> <span class="o">+=</span> <span class="n">np</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span><span class="n">data</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="n">d</span><span class="p">]</span> <span class="o">-</span> <span class="n">centroids</span><span class="p">[</span><span class="n">k</span><span class="p">,</span> <span class="n">d</span><span class="p">])</span><span class="o">**</span><span class="mi">2</span>
|
||
<span class="n">distances</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">dist</span>
|
||
|
||
<span class="c1"># assign each point</span>
|
||
<span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_samples</span><span class="p">):</span>
|
||
<span class="n">smallest</span> <span class="o">=</span> <span class="mf">1e10</span>
|
||
<span class="n">smallest_row_index</span> <span class="o">=</span> <span class="mf">1e10</span>
|
||
<span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_clusters</span><span class="p">):</span>
|
||
<span class="k">if</span> <span class="n">distances</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">]</span> <span class="o"><</span> <span class="n">smallest</span><span class="p">:</span>
|
||
<span class="n">smallest</span> <span class="o">=</span> <span class="n">distances</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">]</span>
|
||
<span class="n">smallest_row_index</span> <span class="o">=</span> <span class="n">k</span>
|
||
|
||
<span class="n">cluster_labels</span><span class="p">[</span><span class="n">n</span><span class="p">]</span> <span class="o">=</span> <span class="n">smallest_row_index</span>
|
||
|
||
<span class="c1"># convergence criteria</span>
|
||
<span class="n">centroid_difference</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span><span class="n">centroids</span> <span class="o">-</span> <span class="n">prev_centroids</span><span class="p">))</span>
|
||
<span class="k">if</span> <span class="n">centroid_difference</span> <span class="o"><</span> <span class="n">tolerance</span><span class="p">:</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">'Converged at iteration </span><span class="si">{</span><span class="n">iteration</span><span class="si">}</span><span class="s1">'</span><span class="p">)</span>
|
||
<span class="k">break</span>
|
||
|
||
<span class="k">elif</span> <span class="n">iteration</span> <span class="o">==</span> <span class="n">max_iterations</span><span class="p">:</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">'Did not converge in </span><span class="si">{</span><span class="n">max_iterations</span><span class="si">}</span><span class="s1"> iterations'</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We now have a simple , un-optimized <span class="math notranslate nohighlight">\(k\)</span>-means
|
||
clustering implementation. Lets plot the final result</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">fig</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">()</span>
|
||
<span class="n">ax</span> <span class="o">=</span> <span class="n">fig</span><span class="o">.</span><span class="n">add_subplot</span><span class="p">()</span>
|
||
<span class="n">unique_cluster_labels</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">unique</span><span class="p">(</span><span class="n">cluster_labels</span><span class="p">)</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">unique_cluster_labels</span><span class="p">:</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">scatter</span><span class="p">(</span><span class="n">data</span><span class="p">[</span><span class="n">cluster_labels</span> <span class="o">==</span> <span class="n">i</span><span class="p">,</span> <span class="mi">0</span><span class="p">],</span>
|
||
<span class="n">data</span><span class="p">[</span><span class="n">cluster_labels</span> <span class="o">==</span> <span class="n">i</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span>
|
||
<span class="n">label</span> <span class="o">=</span> <span class="n">i</span><span class="p">,</span>
|
||
<span class="n">alpha</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">)</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">scatter</span><span class="p">(</span><span class="n">centroids</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">],</span> <span class="n">centroids</span><span class="p">[:,</span> <span class="mi">1</span><span class="p">],</span> <span class="n">c</span><span class="o">=</span><span class="s1">'black'</span><span class="p">)</span>
|
||
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s2">"Final Result of K-means Clustering"</span><span class="p">)</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">naive_kmeans</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">n_clusters</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">max_iterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">tolerance</span><span class="o">=</span><span class="mf">1e-8</span><span class="p">):</span>
|
||
<span class="n">start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
|
||
|
||
<span class="n">n_samples</span><span class="p">,</span> <span class="n">dimensions</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">shape</span>
|
||
<span class="n">n_clusters</span> <span class="o">=</span> <span class="mi">4</span>
|
||
<span class="c1">#np.random.seed(2021)</span>
|
||
<span class="n">centroids</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">n_samples</span><span class="p">,</span> <span class="n">n_clusters</span><span class="p">,</span> <span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span> <span class="p">:]</span>
|
||
<span class="n">distances</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="n">n_samples</span><span class="p">,</span> <span class="n">n_clusters</span><span class="p">))</span>
|
||
|
||
<span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_clusters</span><span class="p">):</span>
|
||
<span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_samples</span><span class="p">):</span>
|
||
<span class="n">dist</span> <span class="o">=</span> <span class="mi">0</span>
|
||
<span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">dimensions</span><span class="p">):</span>
|
||
<span class="n">dist</span> <span class="o">+=</span> <span class="n">np</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span><span class="n">data</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="n">d</span><span class="p">]</span> <span class="o">-</span> <span class="n">centroids</span><span class="p">[</span><span class="n">k</span><span class="p">,</span> <span class="n">d</span><span class="p">])</span><span class="o">**</span><span class="mi">2</span>
|
||
<span class="n">distances</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">dist</span>
|
||
|
||
<span class="n">cluster_labels</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">n_samples</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="s1">'int'</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_samples</span><span class="p">):</span>
|
||
<span class="n">smallest</span> <span class="o">=</span> <span class="mf">1e10</span>
|
||
<span class="n">smallest_row_index</span> <span class="o">=</span> <span class="mf">1e10</span>
|
||
<span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_clusters</span><span class="p">):</span>
|
||
<span class="k">if</span> <span class="n">distances</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">]</span> <span class="o"><</span> <span class="n">smallest</span><span class="p">:</span>
|
||
<span class="n">smallest</span> <span class="o">=</span> <span class="n">distances</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">]</span>
|
||
<span class="n">smallest_row_index</span> <span class="o">=</span> <span class="n">k</span>
|
||
|
||
<span class="n">cluster_labels</span><span class="p">[</span><span class="n">n</span><span class="p">]</span> <span class="o">=</span> <span class="n">smallest_row_index</span>
|
||
|
||
<span class="k">for</span> <span class="n">iteration</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">max_iterations</span><span class="p">):</span>
|
||
<span class="n">prev_centroids</span> <span class="o">=</span> <span class="n">centroids</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||
<span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_clusters</span><span class="p">):</span>
|
||
<span class="n">vector_mean</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">dimensions</span><span class="p">)</span>
|
||
<span class="n">mean_divisor</span> <span class="o">=</span> <span class="mi">0</span>
|
||
<span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_samples</span><span class="p">):</span>
|
||
<span class="k">if</span> <span class="n">cluster_labels</span><span class="p">[</span><span class="n">n</span><span class="p">]</span> <span class="o">==</span> <span class="n">k</span><span class="p">:</span>
|
||
<span class="n">vector_mean</span> <span class="o">+=</span> <span class="n">data</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="p">:]</span>
|
||
<span class="n">mean_divisor</span> <span class="o">+=</span> <span class="mi">1</span>
|
||
|
||
<span class="n">centroids</span><span class="p">[</span><span class="n">k</span><span class="p">,</span> <span class="p">:]</span> <span class="o">=</span> <span class="n">vector_mean</span> <span class="o">/</span> <span class="n">mean_divisor</span>
|
||
|
||
<span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_clusters</span><span class="p">):</span>
|
||
<span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_samples</span><span class="p">):</span>
|
||
<span class="n">dist</span> <span class="o">=</span> <span class="mi">0</span>
|
||
<span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">dimensions</span><span class="p">):</span>
|
||
<span class="n">dist</span> <span class="o">+=</span> <span class="n">np</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span><span class="n">data</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="n">d</span><span class="p">]</span> <span class="o">-</span> <span class="n">centroids</span><span class="p">[</span><span class="n">k</span><span class="p">,</span> <span class="n">d</span><span class="p">])</span><span class="o">**</span><span class="mi">2</span>
|
||
<span class="n">distances</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">dist</span>
|
||
|
||
<span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_samples</span><span class="p">):</span>
|
||
<span class="n">smallest</span> <span class="o">=</span> <span class="mf">1e10</span>
|
||
<span class="n">smallest_row_index</span> <span class="o">=</span> <span class="mf">1e10</span>
|
||
<span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_clusters</span><span class="p">):</span>
|
||
<span class="k">if</span> <span class="n">distances</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">]</span> <span class="o"><</span> <span class="n">smallest</span><span class="p">:</span>
|
||
<span class="n">smallest</span> <span class="o">=</span> <span class="n">distances</span><span class="p">[</span><span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">]</span>
|
||
<span class="n">smallest_row_index</span> <span class="o">=</span> <span class="n">k</span>
|
||
|
||
<span class="n">cluster_labels</span><span class="p">[</span><span class="n">n</span><span class="p">]</span> <span class="o">=</span> <span class="n">smallest_row_index</span>
|
||
|
||
<span class="n">centroid_difference</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span><span class="n">centroids</span> <span class="o">-</span> <span class="n">prev_centroids</span><span class="p">))</span>
|
||
<span class="k">if</span> <span class="n">centroid_difference</span> <span class="o"><</span> <span class="n">tolerance</span><span class="p">:</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">'Converged at iteration </span><span class="si">{</span><span class="n">iteration</span><span class="si">}</span><span class="s1">'</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">'Runtime: </span><span class="si">{</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="n">start_time</span><span class="si">}</span><span class="s1"> seconds'</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">cluster_labels</span><span class="p">,</span> <span class="n">centroids</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">'Did not converge in </span><span class="si">{</span><span class="n">max_iterations</span><span class="si">}</span><span class="s1"> iterations'</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">'Runtime: </span><span class="si">{</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="n">start_time</span><span class="si">}</span><span class="s1"> seconds'</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">cluster_labels</span><span class="p">,</span> <span class="n">centroids</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
kernelName: "python3",
|
||
path: "./."
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</div>
|
||
|
||
|
||
<!-- Previous / next buttons -->
|
||
<div class='prev-next-area'>
|
||
<a class='left-prev' id="prev-link" href="chapter8.html" title="previous page">
|
||
<i class="fas fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title"><span class="section-number">11. </span>Basic ideas of the Principal Component Analysis (PCA)</p>
|
||
</div>
|
||
</a>
|
||
<a class='right-next' id="next-link" href="chapter9.html" title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title"><span class="section-number">13. </span>Neural networks</p>
|
||
</div>
|
||
<i class="fas fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
<footer class="footer">
|
||
<p>
|
||
|
||
By Morten Hjorth-Jensen<br/>
|
||
|
||
© Copyright 2021.<br/>
|
||
</p>
|
||
</footer>
|
||
</main>
|
||
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<script src="_static/js/index.be7d3bbb2ef33a8344ce.js"></script>
|
||
|
||
</body>
|
||
</html> |