updating week44
This commit is contained in:
Morten Hjorth-Jensen
@@ -8,8 +8,8 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 44: From Decision Trees to Bagging methods">
|
||||
<title>Week 44: From Decision Trees to Bagging methods</title>
|
||||
<meta name="description" content="Week 44: Dimensionality Reduction, PCA and Clustering. Decision Trees">
|
||||
<title>Week 44: Dimensionality Reduction, PCA and Clustering. Decision Trees</title>
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<!-- doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week44-bs --no_mako -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -37,7 +37,45 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 44', 2, None, 'overview-of-week-44'),
|
||||
('Thursday', 2, None, 'thursday'),
|
||||
('Thursday, Principal Component Analysis',
|
||||
2,
|
||||
None,
|
||||
'thursday-principal-component-analysis'),
|
||||
('Thursday: Clustering and Unsupervised Learning',
|
||||
2,
|
||||
None,
|
||||
'thursday-clustering-and-unsupervised-learning'),
|
||||
('Basic Idea of the $k$-means Clustering Algorithm',
|
||||
2,
|
||||
None,
|
||||
'basic-idea-of-the-k-means-clustering-algorithm'),
|
||||
('The $k$-means Algorithm', 2, None, 'the-k-means-algorithm'),
|
||||
('Basic Math of the $k$-means Algorithm',
|
||||
2,
|
||||
None,
|
||||
'basic-math-of-the-k-means-algorithm'),
|
||||
('Within Cluster Point Scatter',
|
||||
2,
|
||||
None,
|
||||
'within-cluster-point-scatter'),
|
||||
('More Details', 2, None, 'more-details'),
|
||||
('Total Cluster Variance', 2, None, 'total-cluster-variance'),
|
||||
('The $k$-means Clustering Algorithm',
|
||||
2,
|
||||
None,
|
||||
'the-k-means-clustering-algorithm'),
|
||||
('Summarizing', 2, None, 'summarizing'),
|
||||
('Writing our own Code, the Data Set',
|
||||
2,
|
||||
None,
|
||||
'writing-our-own-code-the-data-set'),
|
||||
('Implementing the $k$-means Algorithm',
|
||||
2,
|
||||
None,
|
||||
'implementing-the-k-means-algorithm'),
|
||||
('Plotting', 2, None, 'plotting'),
|
||||
('Continuing', 2, None, 'continuing'),
|
||||
('Wrapping it up', 2, None, 'wrapping-it-up'),
|
||||
('Decision trees, overarching aims',
|
||||
2,
|
||||
None,
|
||||
@@ -199,7 +237,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week44-bs.html">Week 44: From Decision Trees to Bagging methods</a>
|
||||
<a class="navbar-brand" href="week44-bs.html">Week 44: Dimensionality Reduction, PCA and Clustering. Decision Trees</a>
|
||||
</div>
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
@@ -207,51 +245,65 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs001.html#overview-of-week-44" style="font-size: 80%;">Overview of week 44</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs002.html#thursday" style="font-size: 80%;">Thursday</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs003.html#decision-trees-overarching-aims" style="font-size: 80%;">Decision trees, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs004.html#basics-of-a-tree" style="font-size: 80%;">Basics of a tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs005.html#a-sketch-of-a-tree-regression-problem" style="font-size: 80%;">A Sketch of a Tree, Regression problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs006.html#a-sketch-of-a-tree-classification-problem" style="font-size: 80%;">A Sketch of a Tree, Classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs007.html#a-typical-decision-tree-with-its-pertinent-jargon-classification-problem" style="font-size: 80%;">A typical Decision Tree with its pertinent Jargon, Classification Problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs008.html#general-features" style="font-size: 80%;">General Features</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs009.html#how-do-we-set-it-up" style="font-size: 80%;">How do we set it up?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs010.html#decision-trees-and-regression" style="font-size: 80%;">Decision trees and Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs011.html#building-a-tree-regression" style="font-size: 80%;">Building a tree, regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs012.html#a-top-down-approach-recursive-binary-splitting" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs013.html#making-a-tree" style="font-size: 80%;">Making a tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs014.html#pruning-the-tree" style="font-size: 80%;">Pruning the tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs015.html#cost-complexity-pruning" style="font-size: 80%;">Cost complexity pruning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs016.html#schematic-regression-procedure" style="font-size: 80%;">Schematic Regression Procedure</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs017.html#a-classification-tree" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs018.html#growing-a-classification-tree" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs019.html#classification-tree-how-to-split-nodes" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs020.html#visualizing-the-tree-classification" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs021.html#visualizing-the-tree-the-moons" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs022.html#other-ways-of-visualizing-the-trees" style="font-size: 80%;">Other ways of visualizing the trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs023.html#printing-out-as-text" style="font-size: 80%;">Printing out as text</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs024.html#algorithms-for-setting-up-decision-trees" style="font-size: 80%;">Algorithms for Setting up Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs025.html#the-cart-algorithm-for-classification" style="font-size: 80%;">The CART algorithm for Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs026.html#the-cart-algorithm-for-regression" style="font-size: 80%;">The CART algorithm for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs027.html#computing-the-gini-index" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs028.html#simple-python-code-to-read-in-data-and-perform-classification" style="font-size: 80%;">Simple Python Code to read in Data and perform Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs029.html#computing-the-gini-factor" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs030.html#entropy-and-the-id3-algorithm" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs031.html#cancer-data-again-now-with-decision-trees-and-other-methods" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs032.html#another-example-the-moons-again" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs033.html#playing-around-with-regions" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs034.html#regression-trees" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs035.html#final-regressor-code" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs036.html#pros-and-cons-of-trees-pros" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs037.html#disadvantages" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs038.html#ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods" style="font-size: 80%;">Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs039.html#an-overview-of-ensemble-methods" style="font-size: 80%;">An Overview of Ensemble Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs040.html#bagging" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs041.html#more-bagging" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs042.html#simple-voting-example-head-or-tail" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs043.html#using-the-voting-classifier" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs044.html#please-not-the-moons-again-voting-and-bagging" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs045.html#bagging-examples" style="font-size: 80%;">Bagging Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs046.html#making-your-own-bootstrap-changing-the-level-of-the-decision-tree" style="font-size: 80%;">Making your own Bootstrap: Changing the Level of the Decision Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs002.html#thursday-principal-component-analysis" style="font-size: 80%;">Thursday, Principal Component Analysis</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs003.html#thursday-clustering-and-unsupervised-learning" style="font-size: 80%;">Thursday: Clustering and Unsupervised Learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs004.html#basic-idea-of-the-k-means-clustering-algorithm" style="font-size: 80%;">Basic Idea of the \( k \)-means Clustering Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs005.html#the-k-means-algorithm" style="font-size: 80%;">The \( k \)-means Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs006.html#basic-math-of-the-k-means-algorithm" style="font-size: 80%;">Basic Math of the \( k \)-means Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs007.html#within-cluster-point-scatter" style="font-size: 80%;">Within Cluster Point Scatter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs008.html#more-details" style="font-size: 80%;">More Details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs009.html#total-cluster-variance" style="font-size: 80%;">Total Cluster Variance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs010.html#the-k-means-clustering-algorithm" style="font-size: 80%;">The \( k \)-means Clustering Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs011.html#summarizing" style="font-size: 80%;">Summarizing</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs012.html#writing-our-own-code-the-data-set" style="font-size: 80%;">Writing our own Code, the Data Set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs013.html#implementing-the-k-means-algorithm" style="font-size: 80%;">Implementing the \( k \)-means Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs014.html#plotting" style="font-size: 80%;">Plotting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs015.html#continuing" style="font-size: 80%;">Continuing</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs016.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs017.html#decision-trees-overarching-aims" style="font-size: 80%;">Decision trees, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs018.html#basics-of-a-tree" style="font-size: 80%;">Basics of a tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs019.html#a-sketch-of-a-tree-regression-problem" style="font-size: 80%;">A Sketch of a Tree, Regression problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs020.html#a-sketch-of-a-tree-classification-problem" style="font-size: 80%;">A Sketch of a Tree, Classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs021.html#a-typical-decision-tree-with-its-pertinent-jargon-classification-problem" style="font-size: 80%;">A typical Decision Tree with its pertinent Jargon, Classification Problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs022.html#general-features" style="font-size: 80%;">General Features</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs023.html#how-do-we-set-it-up" style="font-size: 80%;">How do we set it up?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs024.html#decision-trees-and-regression" style="font-size: 80%;">Decision trees and Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs025.html#building-a-tree-regression" style="font-size: 80%;">Building a tree, regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs026.html#a-top-down-approach-recursive-binary-splitting" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs027.html#making-a-tree" style="font-size: 80%;">Making a tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs028.html#pruning-the-tree" style="font-size: 80%;">Pruning the tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs029.html#cost-complexity-pruning" style="font-size: 80%;">Cost complexity pruning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs030.html#schematic-regression-procedure" style="font-size: 80%;">Schematic Regression Procedure</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs031.html#a-classification-tree" style="font-size: 80%;">A Classification Tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs032.html#growing-a-classification-tree" style="font-size: 80%;">Growing a classification tree</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs033.html#classification-tree-how-to-split-nodes" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs034.html#visualizing-the-tree-classification" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs035.html#visualizing-the-tree-the-moons" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs036.html#other-ways-of-visualizing-the-trees" style="font-size: 80%;">Other ways of visualizing the trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs037.html#printing-out-as-text" style="font-size: 80%;">Printing out as text</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs038.html#algorithms-for-setting-up-decision-trees" style="font-size: 80%;">Algorithms for Setting up Decision Trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs039.html#the-cart-algorithm-for-classification" style="font-size: 80%;">The CART algorithm for Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs040.html#the-cart-algorithm-for-regression" style="font-size: 80%;">The CART algorithm for Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs041.html#computing-the-gini-index" style="font-size: 80%;">Computing the Gini index</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs042.html#simple-python-code-to-read-in-data-and-perform-classification" style="font-size: 80%;">Simple Python Code to read in Data and perform Classification</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs043.html#computing-the-gini-factor" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs044.html#entropy-and-the-id3-algorithm" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs045.html#cancer-data-again-now-with-decision-trees-and-other-methods" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs046.html#another-example-the-moons-again" style="font-size: 80%;">Another example, the moons again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs047.html#playing-around-with-regions" style="font-size: 80%;">Playing around with regions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs048.html#regression-trees" style="font-size: 80%;">Regression trees</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs049.html#final-regressor-code" style="font-size: 80%;">Final regressor code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs050.html#pros-and-cons-of-trees-pros" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs051.html#disadvantages" style="font-size: 80%;">Disadvantages</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs052.html#ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods" style="font-size: 80%;">Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs053.html#an-overview-of-ensemble-methods" style="font-size: 80%;">An Overview of Ensemble Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs054.html#bagging" style="font-size: 80%;">Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs055.html#more-bagging" style="font-size: 80%;">More bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs056.html#simple-voting-example-head-or-tail" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs057.html#using-the-voting-classifier" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs058.html#please-not-the-moons-again-voting-and-bagging" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs059.html#bagging-examples" style="font-size: 80%;">Bagging Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week44-bs060.html#making-your-own-bootstrap-changing-the-level-of-the-decision-tree" style="font-size: 80%;">Making your own Bootstrap: Changing the Level of the Decision Tree</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -265,7 +317,7 @@ MathJax.Hub.Config({
|
||||
<!-- ------------------- main content ---------------------- -->
|
||||
<div class="jumbotron">
|
||||
<center>
|
||||
<h1>Week 44: From Decision Trees to Bagging methods</h1>
|
||||
<h1>Week 44: Dimensionality Reduction, PCA and Clustering. Decision Trees</h1>
|
||||
</center> <!-- document title -->
|
||||
|
||||
<!-- author(s): Morten Hjorth-Jensen -->
|
||||
@@ -281,7 +333,7 @@ MathJax.Hub.Config({
|
||||
</center>
|
||||
<br>
|
||||
<center>
|
||||
<h4>Nov 1, 2021</h4>
|
||||
<h4>Nov 2, 2021</h4>
|
||||
</center> <!-- date -->
|
||||
<br>
|
||||
|
||||
@@ -306,7 +358,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._week44-bs008.html">9</a></li>
|
||||
<li><a href="._week44-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week44-bs046.html">47</a></li>
|
||||
<li><a href="._week44-bs060.html">61</a></li>
|
||||
<li><a href="._week44-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -9,8 +9,8 @@ doconce format html week44-reveal.html week44-reveal reveal --html_slide_theme=b
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 44: From Decision Trees to Bagging methods">
|
||||
<title>Week 44: From Decision Trees to Bagging methods</title>
|
||||
<meta name="description" content="Week 44: Dimensionality Reduction, PCA and Clustering. Decision Trees">
|
||||
<title>Week 44: Dimensionality Reduction, PCA and Clustering. Decision Trees</title>
|
||||
|
||||
<!-- reveal.js: https://lab.hakim.se/reveal-js/ -->
|
||||
|
||||
@@ -168,7 +168,7 @@ MathJax.Hub.Config({
|
||||
<section>
|
||||
<!-- ------------------- main content ---------------------- -->
|
||||
<center>
|
||||
<h1 style="text-align: center;">Week 44: From Decision Trees to Bagging methods</h1>
|
||||
<h1 style="text-align: center;">Week 44: Dimensionality Reduction, PCA and Clustering. Decision Trees</h1>
|
||||
</center> <!-- document title -->
|
||||
|
||||
<!-- author(s): Morten Hjorth-Jensen -->
|
||||
@@ -184,7 +184,7 @@ MathJax.Hub.Config({
|
||||
</center>
|
||||
<br>
|
||||
<center>
|
||||
<h4>Nov 1, 2021</h4>
|
||||
<h4>Nov 2, 2021</h4>
|
||||
</center> <!-- date -->
|
||||
<br>
|
||||
|
||||
@@ -198,7 +198,7 @@ MathJax.Hub.Config({
|
||||
<h2 id="overview-of-week-44">Overview of week 44 </h2>
|
||||
|
||||
<ul>
|
||||
<p><li> Thursday: Wrapping up PCA from last week and basics of decision trees, classification and regression algorithms</li>
|
||||
<p><li> Thursday: Wrapping up PCA from last week, Clustering and basics of decision trees, classification and regression algorithms</li>
|
||||
<p><li> Friday: Decision trees, voting models and bagging</li>
|
||||
</ul>
|
||||
<p>
|
||||
@@ -206,9 +206,612 @@ MathJax.Hub.Config({
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="thursday">Thursday </h2>
|
||||
<h2 id="thursday-principal-component-analysis">Thursday, Principal Component Analysis </h2>
|
||||
|
||||
<p>Decision trees and wrapping up PCA.</p>
|
||||
<p>For the principal component analysis,
|
||||
see slides from <a href="https://compphysics.github.io/MachineLearning/doc/pub/week43/html/week43-reveal.html" target="_blank">week 43</a>, in particular from slide 28 and forward
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="thursday-clustering-and-unsupervised-learning">Thursday: Clustering and Unsupervised Learning </h2>
|
||||
|
||||
<p>In general terms cluster analysis, or clustering, is the task of grouping a
|
||||
data-set into different distinct categories based on some measure of equality of
|
||||
the data. This measure is often referred to as a <b>metric</b> or <b>similarity
|
||||
measure</b> in the literature (note: sometimes we deal with a <b>dissimilarity
|
||||
measure</b> instead). Usually, these metrics are formulated as some kind of
|
||||
distance function between points in a high-dimensional space.
|
||||
</p>
|
||||
|
||||
<p>The simplest, and also the most
|
||||
common is the <b>Euclidean distance</b>.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="basic-idea-of-the-k-means-clustering-algorithm">Basic Idea of the \( k \)-means Clustering Algorithm </h2>
|
||||
|
||||
<p>The simplest of all clustering algorithms is the <b>k-means algorithm</b>
|
||||
, sometimes also referred to as <em>Lloyds algorithm</em>. It is the simplest and also
|
||||
the most common. From its simplicity it obtains both strengths and weaknesses.
|
||||
These will be discussed in more detail later. The \( k \)-means algorithm is a
|
||||
<b>centroid based</b> clustering algorithm.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="the-k-means-algorithm">The \( k \)-means Algorithm </h2>
|
||||
|
||||
<p>Assume, we are given \( n \) data points and we wish to split the data into \( K < n \)
|
||||
different categories, or clusters. We label each cluster by an integer
|
||||
</p>
|
||||
|
||||
$$ k\in\{1, \cdots, K \}$.
|
||||
$$
|
||||
|
||||
<p>In the basic k-means algorithm each point is assigned to only
|
||||
one cluster \( k \), and these assignments are <em>non-injective</em> i.e. many-to-one. We
|
||||
can think of these mappings as an encoder \( k = C(i) \), which assigns the \( i \)-th
|
||||
data-point \( \bf x_i \) to the \( k \)-th cluster.
|
||||
</p>
|
||||
|
||||
<p>\( k \)-means algorithm in words:</p>
|
||||
<ol>
|
||||
<p><li> We start with guesses / random initializations of our \( k \) cluster centers / centroids</li>
|
||||
<p><li> For each centroid the points that are most similar are identified</li>
|
||||
<p><li> Then we move / replace each centroid with a coordinate average of all the points that were assigned to that centroid.</li>
|
||||
<p><li> Iterate this points 2, 3) until the centroids no longer move (to some tolerance)</li>
|
||||
</ol>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="basic-math-of-the-k-means-algorithm">Basic Math of the \( k \)-means Algorithm </h2>
|
||||
|
||||
<p>We assume we have \( n \) data-points</p>
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{equation}\tag{1}
|
||||
\boldsymbol{x_i} = \{x_{i, 1}, \cdots, x_{i, p}\}\in\mathbb{R}^p.
|
||||
\end{equation}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>which we wish to group into \( K < n \) clusters. For our dissimilarity measure we
|
||||
use the <em>squared Euclidean distance</em>
|
||||
</p>
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{equation}\tag{2}
|
||||
d(\boldsymbol{x_i}, \boldsymbol{x_i'}) = \sum_{j=1}^p(x_{ij} - x_{i'j})^2
|
||||
= ||\boldsymbol{x_i} - \boldsymbol{x_{i'}}||^2
|
||||
\end{equation}
|
||||
$$
|
||||
<p> <br>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="within-cluster-point-scatter">Within Cluster Point Scatter </h2>
|
||||
|
||||
<p>We define the so called <em>within-cluster point scatter</em> which gives us a
|
||||
measure of how close each data point assigned to the same cluster tends to be to
|
||||
the all the others.
|
||||
</p>
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{equation}\tag{3}
|
||||
W(C) = \frac{1}{2}\sum_{k=1}^K\sum_{C(i)=k}
|
||||
\sum_{C(i')=k}d(\boldsymbol{x_i}, \boldsymbol{x_{i'}}) =
|
||||
\sum_{k=1}^KN_k\sum_{C(i)=k}||\boldsymbol{x_i} - \boldsymbol{\overline{x_k}}||^2
|
||||
\end{equation}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>where \( \boldsymbol{\overline{x_k}} \) is the mean vector associated with the \( k \)-th
|
||||
cluster, and \( N_k = \sum_{i=1}^nI(C(i) = k) \), where the \( I() \) notation is
|
||||
similar to the Kronecker delta (<em>Commonly used in statistics, it just means that
|
||||
when \( i = k \) we have the encoder \( C(i) \)</em>). In other words, the within-cluster
|
||||
scatter measures the compactness of each cluster with respect to the data points
|
||||
assigned to each cluster. This is the quantity that the \( k \)-means algorithm aims
|
||||
to minimize. We refer to this quantity \( W(C) \) as the within cluster scatter
|
||||
because of its relation to the <em>total scatter</em>.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="more-details">More Details </h2>
|
||||
|
||||
<p>We have</p>
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{equation}\tag{4}
|
||||
T = W(C) + B(C) = \frac{1}{2}\sum_{i=1}^n
|
||||
\sum_{i'=1}^nd(\boldsymbol{x_i}, \boldsymbol{x_{i'}})
|
||||
= \frac{1}{2}\sum_{k=1}^K\sum_{C(i)=k}
|
||||
\Big(\sum_{C(i') = k}d(\boldsymbol{x_i}, \boldsymbol{x_{i'}})
|
||||
+ \sum_{C(i')\neq k}d(\boldsymbol{x_i}, \boldsymbol{x_{i'}})\Big).
|
||||
\end{equation}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>This is a quantity that is conserved throughout the \( k \)-means algorithm. It can
|
||||
be thought of as the total amount of information in the data, and it is composed
|
||||
of the aforementioned within-cluster scatter and the <em>between-cluster scatter</em>
|
||||
\( B(C) \). In methods such as principle component analysis the total scatter is not
|
||||
conserved.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="total-cluster-variance">Total Cluster Variance </h2>
|
||||
<p>Given a cluster mean \( \boldsymbol{m_k} \) we define the <b>total cluster variance</b></p>
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{equation}\tag{5}
|
||||
\min_{C, \{\boldsymbol{m_k}\}_1^K}\sum_{k=1}^KN_k\sum||\boldsymbol{x_i} - \boldsymbol{m_k}||^2
|
||||
\end{equation}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>Now we have all the pieces necessary to formally revisit the \( k \)-means algorithm.</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="the-k-means-clustering-algorithm">The \( k \)-means Clustering Algorithm </h2>
|
||||
|
||||
<p>The \( k \)-means clustering algorithm goes as follows </p>
|
||||
|
||||
<ol>
|
||||
<p><li> For a given cluster assignment \( C \), and \( k \) cluster means \( \left{m_1, \cdots, m_k\right} \). We minimize the total cluster variance with respect to the cluster means \( \{m_k\} \) yielding the means of the currently assigned clusters.</li>
|
||||
<p><li> Given a current set of \( k \) means \( \{m_k\} \) the total cluster variance is minimized by assigning each observation to the closest (current) cluster mean. That is <p> <br>
|
||||
$$C(i) = \underset{1\leq k\leq K}{\mathrm{argmin}} ||\boldsymbol{x_i} - \boldsymbol{m_k}||^2$$
|
||||
<p> <br></li>
|
||||
<p><li> Steps 1 and 2 are repeated until the assignments do not change.</li>
|
||||
</ol>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="summarizing">Summarizing </h2>
|
||||
|
||||
<ol>
|
||||
<p><li> Before we start we specify a number \( k \) which is the number of clusters we want to try to separate our data into.</li>
|
||||
<p><li> We initially choose \( k \) random data points in our data as our initial centroids, <em>or means</em> (this is where the name comes from).</li>
|
||||
<p><li> Assign each data point to their closest centroid, based on the squared Euclidean distance.</li>
|
||||
<p><li> For each of the \( k \) cluster we update the centroid by calculating new mean values for all the data points in the cluster.</li>
|
||||
<p><li> Iteratively minimize the within cluster scatter by performing steps (3, 4) until the new assignments stop changing (can be to some tolerance) or until a maximum number of iterations have passed.</li>
|
||||
</ol>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="writing-our-own-code-the-data-set">Writing our own Code, the Data Set </h2>
|
||||
|
||||
<p>Let us now program the most basic version of the algorithm using nothing but
|
||||
Python with numpy arrays. This code is kept intentionally simple to gradually
|
||||
progress our understanding. There is no vectorization of any kind, and even most
|
||||
helper functions are not utilized.
|
||||
</p>
|
||||
|
||||
<p>We need first a dataset to do our cluster analysis on. In our case
|
||||
this is a plain <em>vanilla</em> data set using random numbers using a
|
||||
Gaussian distribution.
|
||||
</p>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">time</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">tensorflow</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">tf</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">matplotlib</span> <span style="color: #8B008B; font-weight: bold">import</span> image
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.cluster</span> <span style="color: #8B008B; font-weight: bold">import</span> KMeans
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">IPython.display</span> <span style="color: #8B008B; font-weight: bold">import</span> display
|
||||
|
||||
np.random.seed(<span style="color: #B452CD">2021</span>)
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>Next we define functions, for ease of use later, to generate Gaussians and to
|
||||
set up our toy data set.
|
||||
</p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">gaussian_points</span>(dim=<span style="color: #B452CD">2</span>, n_points=<span style="color: #B452CD">1000</span>, mean_vector=np.array([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>]),
|
||||
sample_variance=<span style="color: #B452CD">1</span>):
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> Very simple custom function to generate gaussian distributed point clusters</span>
|
||||
<span style="color: #CD5555"> with variable dimension, number of points, means in each direction</span>
|
||||
<span style="color: #CD5555"> (must match dim) and sample variance.</span>
|
||||
|
||||
<span style="color: #CD5555"> Inputs:</span>
|
||||
<span style="color: #CD5555"> dim (int)</span>
|
||||
<span style="color: #CD5555"> n_points (int)</span>
|
||||
<span style="color: #CD5555"> mean_vector (np.array) (where index 0 is x, index 1 is y etc.)</span>
|
||||
<span style="color: #CD5555"> sample_variance (float)</span>
|
||||
|
||||
<span style="color: #CD5555"> Returns:</span>
|
||||
<span style="color: #CD5555"> data (np.array): with dimensions (dim x n_points)</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
|
||||
mean_matrix = np.zeros(dim) + mean_vector
|
||||
covariance_matrix = np.eye(dim) * sample_variance
|
||||
data = np.random.multivariate_normal(mean_matrix, covariance_matrix,
|
||||
n_points)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> data
|
||||
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">generate_simple_clustering_dataset</span>(dim=<span style="color: #B452CD">2</span>, n_points=<span style="color: #B452CD">1000</span>, plotting=<span style="color: #8B008B; font-weight: bold">True</span>,
|
||||
return_data=<span style="color: #8B008B; font-weight: bold">True</span>):
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> Toy model to illustrate k-means clustering</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
|
||||
data1 = gaussian_points(mean_vector=np.array([<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>]))
|
||||
data2 = gaussian_points()
|
||||
data3 = gaussian_points(mean_vector=np.array([<span style="color: #B452CD">1</span>, <span style="color: #B452CD">4.5</span>]))
|
||||
data4 = gaussian_points(mean_vector=np.array([<span style="color: #B452CD">5</span>, <span style="color: #B452CD">1</span>]))
|
||||
data = np.concatenate((data1, data2, data3, data4), axis=<span style="color: #B452CD">0</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> plotting:
|
||||
fig, ax = plt.subplots()
|
||||
ax.scatter(data[:, <span style="color: #B452CD">0</span>], data[:, <span style="color: #B452CD">1</span>], alpha=<span style="color: #B452CD">0.2</span>)
|
||||
ax.set_title(<span style="color: #CD5555">'Toy Model Dataset'</span>)
|
||||
plt.show()
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> return_data:
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> data
|
||||
|
||||
|
||||
data = generate_simple_clustering_dataset()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="implementing-the-k-means-algorithm">Implementing the \( k \)-means Algorithm </h2>
|
||||
|
||||
<p>With the above dataset we start
|
||||
implementing the \( k \)-means algorithm.
|
||||
</p>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="font-size: 80%; line-height: 125%;">n_samples, dimensions = data.shape
|
||||
n_clusters = <span style="color: #B452CD">4</span>
|
||||
|
||||
<span style="color: #228B22"># we randomly initialize our centroids</span>
|
||||
np.random.seed(<span style="color: #B452CD">2021</span>)
|
||||
centroids = data[np.random.choice(n_samples, n_clusters, replace=<span style="color: #8B008B; font-weight: bold">False</span>), :]
|
||||
distances = np.zeros((n_samples, n_clusters))
|
||||
|
||||
<span style="color: #228B22"># first we need to calculate the distance to each centroid from our data</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
dist = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> d <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(dimensions):
|
||||
dist += np.abs(data[n, d] - centroids[k, d])**<span style="color: #B452CD">2</span>
|
||||
distances[n, k] = dist
|
||||
|
||||
<span style="color: #228B22"># we initialize an array to keep track of to which cluster each point belongs</span>
|
||||
<span style="color: #228B22"># the way we set it up here the index tracks which point and the value which</span>
|
||||
<span style="color: #228B22"># cluster the point belongs to</span>
|
||||
cluster_labels = np.zeros(n_samples, dtype=<span style="color: #CD5555">'int'</span>)
|
||||
|
||||
<span style="color: #228B22"># next we loop through our samples and for every point assign it to the cluster</span>
|
||||
<span style="color: #228B22"># to which it has the smallest distance to</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
<span style="color: #228B22"># tracking variables (all of this is basically just an argmin)</span>
|
||||
smallest = <span style="color: #B452CD">1e10</span>
|
||||
smallest_row_index = <span style="color: #B452CD">1e10</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> distances[n, k] < smallest:
|
||||
smallest = distances[n, k]
|
||||
smallest_row_index = k
|
||||
|
||||
cluster_labels[n] = smallest_row_index
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="plotting">Plotting </h2>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="font-size: 80%; line-height: 125%;">fig = plt.figure()
|
||||
ax = fig.add_subplot()
|
||||
unique_cluster_labels = np.unique(cluster_labels)
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> unique_cluster_labels:
|
||||
ax.scatter(data[cluster_labels == i, <span style="color: #B452CD">0</span>],
|
||||
data[cluster_labels == i, <span style="color: #B452CD">1</span>],
|
||||
label = i,
|
||||
alpha = <span style="color: #B452CD">0.2</span>)
|
||||
ax.scatter(centroids[:, <span style="color: #B452CD">0</span>], centroids[:, <span style="color: #B452CD">1</span>], c=<span style="color: #CD5555">'black'</span>)
|
||||
|
||||
ax.set_title(<span style="color: #CD5555">"First Grouping of Points to Centroids"</span>)
|
||||
|
||||
plt.show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>So what do we have so far? We have 'picked' \( k \) centroids at random from our
|
||||
data points. There are other ways of more intelligently choosing their
|
||||
initializations, however for our purposes randomly is fine. Then we have
|
||||
initialized an array 'distances' which holds the information of the distance,
|
||||
<em>or dissimilarity</em>, of every point to of our centroids. Finally, we have
|
||||
initialized an array 'cluster_labels' which according to our distances array
|
||||
holds the information of to which centroid every point is assigned. This was the
|
||||
first pass of our algorithm. Essentially, all we need to do now is repeat the
|
||||
distance and assignment steps above until we have reached a desired convergence
|
||||
or a maximum amount of iterations.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="continuing">Continuing </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="font-size: 80%; line-height: 125%;">max_iterations = <span style="color: #B452CD">100</span>
|
||||
tolerance = <span style="color: #B452CD">1e-8</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> iteration <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(max_iterations):
|
||||
prev_centroids = centroids.copy()
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #228B22"># this array will be used to update our centroid positions</span>
|
||||
vector_mean = np.zeros(dimensions)
|
||||
mean_divisor = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> cluster_labels[n] == k:
|
||||
vector_mean += data[n, :]
|
||||
mean_divisor += <span style="color: #B452CD">1</span>
|
||||
|
||||
<span style="color: #228B22"># update according to the k means</span>
|
||||
centroids[k, :] = vector_mean / mean_divisor
|
||||
|
||||
<span style="color: #228B22"># we find the dissimilarity</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
dist = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> d <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(dimensions):
|
||||
dist += np.abs(data[n, d] - centroids[k, d])**<span style="color: #B452CD">2</span>
|
||||
distances[n, k] = dist
|
||||
|
||||
<span style="color: #228B22"># assign each point</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
smallest = <span style="color: #B452CD">1e10</span>
|
||||
smallest_row_index = <span style="color: #B452CD">1e10</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> distances[n, k] < smallest:
|
||||
smallest = distances[n, k]
|
||||
smallest_row_index = k
|
||||
|
||||
cluster_labels[n] = smallest_row_index
|
||||
|
||||
<span style="color: #228B22"># convergence criteria</span>
|
||||
centroid_difference = np.sum(np.abs(centroids - prev_centroids))
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> centroid_difference < tolerance:
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f'Converged at iteration {</span>iteration<span style="color: #CD5555">}'</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">break</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">elif</span> iteration == max_iterations:
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f'Did not converge in {</span>max_iterations<span style="color: #CD5555">} iterations'</span>)
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="wrapping-it-up">Wrapping it up </h2>
|
||||
<p>We now have a simple , un-optimized \( k \)-means
|
||||
clustering implementation. Lets plot the final result
|
||||
</p>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="font-size: 80%; line-height: 125%;">fig = plt.figure()
|
||||
ax = fig.add_subplot()
|
||||
unique_cluster_labels = np.unique(cluster_labels)
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> unique_cluster_labels:
|
||||
ax.scatter(data[cluster_labels == i, <span style="color: #B452CD">0</span>],
|
||||
data[cluster_labels == i, <span style="color: #B452CD">1</span>],
|
||||
label = i,
|
||||
alpha = <span style="color: #B452CD">0.2</span>)
|
||||
ax.scatter(centroids[:, <span style="color: #B452CD">0</span>], centroids[:, <span style="color: #B452CD">1</span>], c=<span style="color: #CD5555">'black'</span>)
|
||||
|
||||
ax.set_title(<span style="color: #CD5555">"Final Result of K-means Clustering"</span>)
|
||||
|
||||
plt.show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">naive_kmeans</span>(data, n_clusters=<span style="color: #B452CD">4</span>, max_iterations=<span style="color: #B452CD">100</span>, tolerance=<span style="color: #B452CD">1e-8</span>):
|
||||
start_time = time.time()
|
||||
|
||||
n_samples, dimensions = data.shape
|
||||
n_clusters = <span style="color: #B452CD">4</span>
|
||||
<span style="color: #228B22">#np.random.seed(2021)</span>
|
||||
centroids = data[np.random.choice(n_samples, n_clusters, replace=<span style="color: #8B008B; font-weight: bold">False</span>), :]
|
||||
distances = np.zeros((n_samples, n_clusters))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
dist = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> d <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(dimensions):
|
||||
dist += np.abs(data[n, d] - centroids[k, d])**<span style="color: #B452CD">2</span>
|
||||
distances[n, k] = dist
|
||||
|
||||
cluster_labels = np.zeros(n_samples, dtype=<span style="color: #CD5555">'int'</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
smallest = <span style="color: #B452CD">1e10</span>
|
||||
smallest_row_index = <span style="color: #B452CD">1e10</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> distances[n, k] < smallest:
|
||||
smallest = distances[n, k]
|
||||
smallest_row_index = k
|
||||
|
||||
cluster_labels[n] = smallest_row_index
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> iteration <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(max_iterations):
|
||||
prev_centroids = centroids.copy()
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
vector_mean = np.zeros(dimensions)
|
||||
mean_divisor = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> cluster_labels[n] == k:
|
||||
vector_mean += data[n, :]
|
||||
mean_divisor += <span style="color: #B452CD">1</span>
|
||||
|
||||
centroids[k, :] = vector_mean / mean_divisor
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
dist = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> d <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(dimensions):
|
||||
dist += np.abs(data[n, d] - centroids[k, d])**<span style="color: #B452CD">2</span>
|
||||
distances[n, k] = dist
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
smallest = <span style="color: #B452CD">1e10</span>
|
||||
smallest_row_index = <span style="color: #B452CD">1e10</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> distances[n, k] < smallest:
|
||||
smallest = distances[n, k]
|
||||
smallest_row_index = k
|
||||
|
||||
cluster_labels[n] = smallest_row_index
|
||||
|
||||
centroid_difference = np.sum(np.abs(centroids - prev_centroids))
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> centroid_difference < tolerance:
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f'Converged at iteration {</span>iteration<span style="color: #CD5555">}'</span>)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f'Runtime: {</span>time.time() - start_time<span style="color: #CD5555">} seconds'</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> cluster_labels, centroids
|
||||
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f'Did not converge in {</span>max_iterations<span style="color: #CD5555">} iterations'</span>)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f'Runtime: {</span>time.time() - start_time<span style="color: #CD5555">} seconds'</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> cluster_labels, centroids
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
|
||||
@@ -8,8 +8,8 @@ doconce format html week44.do.txt --pygments_html_style=perldoc --html_style=sol
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 44: From Decision Trees to Bagging methods">
|
||||
<title>Week 44: From Decision Trees to Bagging methods</title>
|
||||
<meta name="description" content="Week 44: Dimensionality Reduction, PCA and Clustering. Decision Trees">
|
||||
<title>Week 44: Dimensionality Reduction, PCA and Clustering. Decision Trees</title>
|
||||
<link href="https://cdn.rawgit.com/doconce/doconce/master/bundled/html_styles/style_solarized_box/css/solarized_light_code.css" rel="stylesheet" type="text/css" title="light"/>
|
||||
<script src="https://cdn.rawgit.com/doconce/doconce/master/bundled/html_styles/style_solarized_box/js/highlight.pack.js"></script>
|
||||
<script>hljs.initHighlightingOnLoad();</script>
|
||||
@@ -64,7 +64,45 @@ div.toc p,a {
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 44', 2, None, 'overview-of-week-44'),
|
||||
('Thursday', 2, None, 'thursday'),
|
||||
('Thursday, Principal Component Analysis',
|
||||
2,
|
||||
None,
|
||||
'thursday-principal-component-analysis'),
|
||||
('Thursday: Clustering and Unsupervised Learning',
|
||||
2,
|
||||
None,
|
||||
'thursday-clustering-and-unsupervised-learning'),
|
||||
('Basic Idea of the $k$-means Clustering Algorithm',
|
||||
2,
|
||||
None,
|
||||
'basic-idea-of-the-k-means-clustering-algorithm'),
|
||||
('The $k$-means Algorithm', 2, None, 'the-k-means-algorithm'),
|
||||
('Basic Math of the $k$-means Algorithm',
|
||||
2,
|
||||
None,
|
||||
'basic-math-of-the-k-means-algorithm'),
|
||||
('Within Cluster Point Scatter',
|
||||
2,
|
||||
None,
|
||||
'within-cluster-point-scatter'),
|
||||
('More Details', 2, None, 'more-details'),
|
||||
('Total Cluster Variance', 2, None, 'total-cluster-variance'),
|
||||
('The $k$-means Clustering Algorithm',
|
||||
2,
|
||||
None,
|
||||
'the-k-means-clustering-algorithm'),
|
||||
('Summarizing', 2, None, 'summarizing'),
|
||||
('Writing our own Code, the Data Set',
|
||||
2,
|
||||
None,
|
||||
'writing-our-own-code-the-data-set'),
|
||||
('Implementing the $k$-means Algorithm',
|
||||
2,
|
||||
None,
|
||||
'implementing-the-k-means-algorithm'),
|
||||
('Plotting', 2, None, 'plotting'),
|
||||
('Continuing', 2, None, 'continuing'),
|
||||
('Wrapping it up', 2, None, 'wrapping-it-up'),
|
||||
('Decision trees, overarching aims',
|
||||
2,
|
||||
None,
|
||||
@@ -220,7 +258,7 @@ MathJax.Hub.Config({
|
||||
|
||||
<!-- ------------------- main content ---------------------- -->
|
||||
<center>
|
||||
<h1>Week 44: From Decision Trees to Bagging methods</h1>
|
||||
<h1>Week 44: Dimensionality Reduction, PCA and Clustering. Decision Trees</h1>
|
||||
</center> <!-- document title -->
|
||||
|
||||
<!-- author(s): Morten Hjorth-Jensen -->
|
||||
@@ -236,7 +274,7 @@ MathJax.Hub.Config({
|
||||
</center>
|
||||
<br>
|
||||
<center>
|
||||
<h4>Nov 1, 2021</h4>
|
||||
<h4>Nov 2, 2021</h4>
|
||||
</center> <!-- date -->
|
||||
<br>
|
||||
|
||||
@@ -244,15 +282,594 @@ MathJax.Hub.Config({
|
||||
<h2 id="overview-of-week-44">Overview of week 44 </h2>
|
||||
|
||||
<ul>
|
||||
<li> Thursday: Wrapping up PCA from last week and basics of decision trees, classification and regression algorithms</li>
|
||||
<li> Thursday: Wrapping up PCA from last week, Clustering and basics of decision trees, classification and regression algorithms</li>
|
||||
<li> Friday: Decision trees, voting models and bagging</li>
|
||||
</ul>
|
||||
<p>Geron's chapter 6 covers decision trees while ensemble models, voting and bagging are discussed in chapter 7. See also lecture from <a href="https://www.uio.no/studier/emner/matnat/math/STK-IN4300/h20/slides/lecture_7.pdf" target="_blank">STK-IN4300, lecture 7</a>. Chapter 9.2 of Hastie et al contains also a good discussion.</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="thursday">Thursday </h2>
|
||||
<h2 id="thursday-principal-component-analysis">Thursday, Principal Component Analysis </h2>
|
||||
|
||||
<p>For the principal component analysis,
|
||||
see slides from <a href="https://compphysics.github.io/MachineLearning/doc/pub/week43/html/week43-reveal.html" target="_blank">week 43</a>, in particular from slide 28 and forward
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="thursday-clustering-and-unsupervised-learning">Thursday: Clustering and Unsupervised Learning </h2>
|
||||
|
||||
<p>In general terms cluster analysis, or clustering, is the task of grouping a
|
||||
data-set into different distinct categories based on some measure of equality of
|
||||
the data. This measure is often referred to as a <b>metric</b> or <b>similarity
|
||||
measure</b> in the literature (note: sometimes we deal with a <b>dissimilarity
|
||||
measure</b> instead). Usually, these metrics are formulated as some kind of
|
||||
distance function between points in a high-dimensional space.
|
||||
</p>
|
||||
|
||||
<p>The simplest, and also the most
|
||||
common is the <b>Euclidean distance</b>.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="basic-idea-of-the-k-means-clustering-algorithm">Basic Idea of the \( k \)-means Clustering Algorithm </h2>
|
||||
|
||||
<p>The simplest of all clustering algorithms is the <b>k-means algorithm</b>
|
||||
, sometimes also referred to as <em>Lloyds algorithm</em>. It is the simplest and also
|
||||
the most common. From its simplicity it obtains both strengths and weaknesses.
|
||||
These will be discussed in more detail later. The \( k \)-means algorithm is a
|
||||
<b>centroid based</b> clustering algorithm.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="the-k-means-algorithm">The \( k \)-means Algorithm </h2>
|
||||
|
||||
<p>Assume, we are given \( n \) data points and we wish to split the data into \( K < n \)
|
||||
different categories, or clusters. We label each cluster by an integer
|
||||
</p>
|
||||
|
||||
$$ k\in\{1, \cdots, K \}$.
|
||||
$$
|
||||
|
||||
<p>In the basic k-means algorithm each point is assigned to only
|
||||
one cluster \( k \), and these assignments are <em>non-injective</em> i.e. many-to-one. We
|
||||
can think of these mappings as an encoder \( k = C(i) \), which assigns the \( i \)-th
|
||||
data-point \( \bf x_i \) to the \( k \)-th cluster.
|
||||
</p>
|
||||
|
||||
<p>\( k \)-means algorithm in words:</p>
|
||||
<ol>
|
||||
<li> We start with guesses / random initializations of our \( k \) cluster centers / centroids</li>
|
||||
<li> For each centroid the points that are most similar are identified</li>
|
||||
<li> Then we move / replace each centroid with a coordinate average of all the points that were assigned to that centroid.</li>
|
||||
<li> Iterate this points 2, 3) until the centroids no longer move (to some tolerance)</li>
|
||||
</ol>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="basic-math-of-the-k-means-algorithm">Basic Math of the \( k \)-means Algorithm </h2>
|
||||
|
||||
<p>We assume we have \( n \) data-points</p>
|
||||
$$
|
||||
\begin{equation}\label{eq:kmeanspoints}
|
||||
\boldsymbol{x_i} = \{x_{i, 1}, \cdots, x_{i, p}\}\in\mathbb{R}^p.
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>which we wish to group into \( K < n \) clusters. For our dissimilarity measure we
|
||||
use the <em>squared Euclidean distance</em>
|
||||
</p>
|
||||
$$
|
||||
\begin{equation}\label{eq:squaredeuclidean}
|
||||
d(\boldsymbol{x_i}, \boldsymbol{x_i'}) = \sum_{j=1}^p(x_{ij} - x_{i'j})^2
|
||||
= ||\boldsymbol{x_i} - \boldsymbol{x_{i'}}||^2
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="within-cluster-point-scatter">Within Cluster Point Scatter </h2>
|
||||
|
||||
<p>We define the so called <em>within-cluster point scatter</em> which gives us a
|
||||
measure of how close each data point assigned to the same cluster tends to be to
|
||||
the all the others.
|
||||
</p>
|
||||
$$
|
||||
\begin{equation}\label{eq:withincluster}
|
||||
W(C) = \frac{1}{2}\sum_{k=1}^K\sum_{C(i)=k}
|
||||
\sum_{C(i')=k}d(\boldsymbol{x_i}, \boldsymbol{x_{i'}}) =
|
||||
\sum_{k=1}^KN_k\sum_{C(i)=k}||\boldsymbol{x_i} - \boldsymbol{\overline{x_k}}||^2
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>where \( \boldsymbol{\overline{x_k}} \) is the mean vector associated with the \( k \)-th
|
||||
cluster, and \( N_k = \sum_{i=1}^nI(C(i) = k) \), where the \( I() \) notation is
|
||||
similar to the Kronecker delta (<em>Commonly used in statistics, it just means that
|
||||
when \( i = k \) we have the encoder \( C(i) \)</em>). In other words, the within-cluster
|
||||
scatter measures the compactness of each cluster with respect to the data points
|
||||
assigned to each cluster. This is the quantity that the \( k \)-means algorithm aims
|
||||
to minimize. We refer to this quantity \( W(C) \) as the within cluster scatter
|
||||
because of its relation to the <em>total scatter</em>.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="more-details">More Details </h2>
|
||||
|
||||
<p>We have</p>
|
||||
$$
|
||||
\begin{equation}\label{eq:totalscatter}
|
||||
T = W(C) + B(C) = \frac{1}{2}\sum_{i=1}^n
|
||||
\sum_{i'=1}^nd(\boldsymbol{x_i}, \boldsymbol{x_{i'}})
|
||||
= \frac{1}{2}\sum_{k=1}^K\sum_{C(i)=k}
|
||||
\Big(\sum_{C(i') = k}d(\boldsymbol{x_i}, \boldsymbol{x_{i'}})
|
||||
+ \sum_{C(i')\neq k}d(\boldsymbol{x_i}, \boldsymbol{x_{i'}})\Big).
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>This is a quantity that is conserved throughout the \( k \)-means algorithm. It can
|
||||
be thought of as the total amount of information in the data, and it is composed
|
||||
of the aforementioned within-cluster scatter and the <em>between-cluster scatter</em>
|
||||
\( B(C) \). In methods such as principle component analysis the total scatter is not
|
||||
conserved.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="total-cluster-variance">Total Cluster Variance </h2>
|
||||
<p>Given a cluster mean \( \boldsymbol{m_k} \) we define the <b>total cluster variance</b></p>
|
||||
$$
|
||||
\begin{equation}\label{eq:totalclustervariance}
|
||||
\min_{C, \{\boldsymbol{m_k}\}_1^K}\sum_{k=1}^KN_k\sum||\boldsymbol{x_i} - \boldsymbol{m_k}||^2
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>Now we have all the pieces necessary to formally revisit the \( k \)-means algorithm.</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="the-k-means-clustering-algorithm">The \( k \)-means Clustering Algorithm </h2>
|
||||
|
||||
<p>The \( k \)-means clustering algorithm goes as follows </p>
|
||||
|
||||
<ol>
|
||||
<li> For a given cluster assignment \( C \), and \( k \) cluster means \( \left{m_1, \cdots, m_k\right} \). We minimize the total cluster variance with respect to the cluster means \( \{m_k\} \) yielding the means of the currently assigned clusters.</li>
|
||||
<li> Given a current set of \( k \) means \( \{m_k\} \) the total cluster variance is minimized by assigning each observation to the closest (current) cluster mean. That is $$C(i) = \underset{1\leq k\leq K}{\mathrm{argmin}} ||\boldsymbol{x_i} - \boldsymbol{m_k}||^2$$</li>
|
||||
<li> Steps 1 and 2 are repeated until the assignments do not change.</li>
|
||||
</ol>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="summarizing">Summarizing </h2>
|
||||
|
||||
<ol>
|
||||
<li> Before we start we specify a number \( k \) which is the number of clusters we want to try to separate our data into.</li>
|
||||
<li> We initially choose \( k \) random data points in our data as our initial centroids, <em>or means</em> (this is where the name comes from).</li>
|
||||
<li> Assign each data point to their closest centroid, based on the squared Euclidean distance.</li>
|
||||
<li> For each of the \( k \) cluster we update the centroid by calculating new mean values for all the data points in the cluster.</li>
|
||||
<li> Iteratively minimize the within cluster scatter by performing steps (3, 4) until the new assignments stop changing (can be to some tolerance) or until a maximum number of iterations have passed.</li>
|
||||
</ol>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="writing-our-own-code-the-data-set">Writing our own Code, the Data Set </h2>
|
||||
|
||||
<p>Let us now program the most basic version of the algorithm using nothing but
|
||||
Python with numpy arrays. This code is kept intentionally simple to gradually
|
||||
progress our understanding. There is no vectorization of any kind, and even most
|
||||
helper functions are not utilized.
|
||||
</p>
|
||||
|
||||
<p>We need first a dataset to do our cluster analysis on. In our case
|
||||
this is a plain <em>vanilla</em> data set using random numbers using a
|
||||
Gaussian distribution.
|
||||
</p>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">time</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">tensorflow</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">tf</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">matplotlib</span> <span style="color: #8B008B; font-weight: bold">import</span> image
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.cluster</span> <span style="color: #8B008B; font-weight: bold">import</span> KMeans
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">IPython.display</span> <span style="color: #8B008B; font-weight: bold">import</span> display
|
||||
|
||||
np.random.seed(<span style="color: #B452CD">2021</span>)
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>Next we define functions, for ease of use later, to generate Gaussians and to
|
||||
set up our toy data set.
|
||||
</p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">gaussian_points</span>(dim=<span style="color: #B452CD">2</span>, n_points=<span style="color: #B452CD">1000</span>, mean_vector=np.array([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>]),
|
||||
sample_variance=<span style="color: #B452CD">1</span>):
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> Very simple custom function to generate gaussian distributed point clusters</span>
|
||||
<span style="color: #CD5555"> with variable dimension, number of points, means in each direction</span>
|
||||
<span style="color: #CD5555"> (must match dim) and sample variance.</span>
|
||||
|
||||
<span style="color: #CD5555"> Inputs:</span>
|
||||
<span style="color: #CD5555"> dim (int)</span>
|
||||
<span style="color: #CD5555"> n_points (int)</span>
|
||||
<span style="color: #CD5555"> mean_vector (np.array) (where index 0 is x, index 1 is y etc.)</span>
|
||||
<span style="color: #CD5555"> sample_variance (float)</span>
|
||||
|
||||
<span style="color: #CD5555"> Returns:</span>
|
||||
<span style="color: #CD5555"> data (np.array): with dimensions (dim x n_points)</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
|
||||
mean_matrix = np.zeros(dim) + mean_vector
|
||||
covariance_matrix = np.eye(dim) * sample_variance
|
||||
data = np.random.multivariate_normal(mean_matrix, covariance_matrix,
|
||||
n_points)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> data
|
||||
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">generate_simple_clustering_dataset</span>(dim=<span style="color: #B452CD">2</span>, n_points=<span style="color: #B452CD">1000</span>, plotting=<span style="color: #8B008B; font-weight: bold">True</span>,
|
||||
return_data=<span style="color: #8B008B; font-weight: bold">True</span>):
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> Toy model to illustrate k-means clustering</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
|
||||
data1 = gaussian_points(mean_vector=np.array([<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>]))
|
||||
data2 = gaussian_points()
|
||||
data3 = gaussian_points(mean_vector=np.array([<span style="color: #B452CD">1</span>, <span style="color: #B452CD">4.5</span>]))
|
||||
data4 = gaussian_points(mean_vector=np.array([<span style="color: #B452CD">5</span>, <span style="color: #B452CD">1</span>]))
|
||||
data = np.concatenate((data1, data2, data3, data4), axis=<span style="color: #B452CD">0</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> plotting:
|
||||
fig, ax = plt.subplots()
|
||||
ax.scatter(data[:, <span style="color: #B452CD">0</span>], data[:, <span style="color: #B452CD">1</span>], alpha=<span style="color: #B452CD">0.2</span>)
|
||||
ax.set_title(<span style="color: #CD5555">'Toy Model Dataset'</span>)
|
||||
plt.show()
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> return_data:
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> data
|
||||
|
||||
|
||||
data = generate_simple_clustering_dataset()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="implementing-the-k-means-algorithm">Implementing the \( k \)-means Algorithm </h2>
|
||||
|
||||
<p>With the above dataset we start
|
||||
implementing the \( k \)-means algorithm.
|
||||
</p>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="line-height: 125%;">n_samples, dimensions = data.shape
|
||||
n_clusters = <span style="color: #B452CD">4</span>
|
||||
|
||||
<span style="color: #228B22"># we randomly initialize our centroids</span>
|
||||
np.random.seed(<span style="color: #B452CD">2021</span>)
|
||||
centroids = data[np.random.choice(n_samples, n_clusters, replace=<span style="color: #8B008B; font-weight: bold">False</span>), :]
|
||||
distances = np.zeros((n_samples, n_clusters))
|
||||
|
||||
<span style="color: #228B22"># first we need to calculate the distance to each centroid from our data</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
dist = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> d <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(dimensions):
|
||||
dist += np.abs(data[n, d] - centroids[k, d])**<span style="color: #B452CD">2</span>
|
||||
distances[n, k] = dist
|
||||
|
||||
<span style="color: #228B22"># we initialize an array to keep track of to which cluster each point belongs</span>
|
||||
<span style="color: #228B22"># the way we set it up here the index tracks which point and the value which</span>
|
||||
<span style="color: #228B22"># cluster the point belongs to</span>
|
||||
cluster_labels = np.zeros(n_samples, dtype=<span style="color: #CD5555">'int'</span>)
|
||||
|
||||
<span style="color: #228B22"># next we loop through our samples and for every point assign it to the cluster</span>
|
||||
<span style="color: #228B22"># to which it has the smallest distance to</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
<span style="color: #228B22"># tracking variables (all of this is basically just an argmin)</span>
|
||||
smallest = <span style="color: #B452CD">1e10</span>
|
||||
smallest_row_index = <span style="color: #B452CD">1e10</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> distances[n, k] < smallest:
|
||||
smallest = distances[n, k]
|
||||
smallest_row_index = k
|
||||
|
||||
cluster_labels[n] = smallest_row_index
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="plotting">Plotting </h2>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="line-height: 125%;">fig = plt.figure()
|
||||
ax = fig.add_subplot()
|
||||
unique_cluster_labels = np.unique(cluster_labels)
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> unique_cluster_labels:
|
||||
ax.scatter(data[cluster_labels == i, <span style="color: #B452CD">0</span>],
|
||||
data[cluster_labels == i, <span style="color: #B452CD">1</span>],
|
||||
label = i,
|
||||
alpha = <span style="color: #B452CD">0.2</span>)
|
||||
ax.scatter(centroids[:, <span style="color: #B452CD">0</span>], centroids[:, <span style="color: #B452CD">1</span>], c=<span style="color: #CD5555">'black'</span>)
|
||||
|
||||
ax.set_title(<span style="color: #CD5555">"First Grouping of Points to Centroids"</span>)
|
||||
|
||||
plt.show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>So what do we have so far? We have 'picked' \( k \) centroids at random from our
|
||||
data points. There are other ways of more intelligently choosing their
|
||||
initializations, however for our purposes randomly is fine. Then we have
|
||||
initialized an array 'distances' which holds the information of the distance,
|
||||
<em>or dissimilarity</em>, of every point to of our centroids. Finally, we have
|
||||
initialized an array 'cluster_labels' which according to our distances array
|
||||
holds the information of to which centroid every point is assigned. This was the
|
||||
first pass of our algorithm. Essentially, all we need to do now is repeat the
|
||||
distance and assignment steps above until we have reached a desired convergence
|
||||
or a maximum amount of iterations.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="continuing">Continuing </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="line-height: 125%;">max_iterations = <span style="color: #B452CD">100</span>
|
||||
tolerance = <span style="color: #B452CD">1e-8</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> iteration <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(max_iterations):
|
||||
prev_centroids = centroids.copy()
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #228B22"># this array will be used to update our centroid positions</span>
|
||||
vector_mean = np.zeros(dimensions)
|
||||
mean_divisor = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> cluster_labels[n] == k:
|
||||
vector_mean += data[n, :]
|
||||
mean_divisor += <span style="color: #B452CD">1</span>
|
||||
|
||||
<span style="color: #228B22"># update according to the k means</span>
|
||||
centroids[k, :] = vector_mean / mean_divisor
|
||||
|
||||
<span style="color: #228B22"># we find the dissimilarity</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
dist = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> d <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(dimensions):
|
||||
dist += np.abs(data[n, d] - centroids[k, d])**<span style="color: #B452CD">2</span>
|
||||
distances[n, k] = dist
|
||||
|
||||
<span style="color: #228B22"># assign each point</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
smallest = <span style="color: #B452CD">1e10</span>
|
||||
smallest_row_index = <span style="color: #B452CD">1e10</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> distances[n, k] < smallest:
|
||||
smallest = distances[n, k]
|
||||
smallest_row_index = k
|
||||
|
||||
cluster_labels[n] = smallest_row_index
|
||||
|
||||
<span style="color: #228B22"># convergence criteria</span>
|
||||
centroid_difference = np.sum(np.abs(centroids - prev_centroids))
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> centroid_difference < tolerance:
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f'Converged at iteration {</span>iteration<span style="color: #CD5555">}'</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">break</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">elif</span> iteration == max_iterations:
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f'Did not converge in {</span>max_iterations<span style="color: #CD5555">} iterations'</span>)
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="wrapping-it-up">Wrapping it up </h2>
|
||||
<p>We now have a simple , un-optimized \( k \)-means
|
||||
clustering implementation. Lets plot the final result
|
||||
</p>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="line-height: 125%;">fig = plt.figure()
|
||||
ax = fig.add_subplot()
|
||||
unique_cluster_labels = np.unique(cluster_labels)
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> unique_cluster_labels:
|
||||
ax.scatter(data[cluster_labels == i, <span style="color: #B452CD">0</span>],
|
||||
data[cluster_labels == i, <span style="color: #B452CD">1</span>],
|
||||
label = i,
|
||||
alpha = <span style="color: #B452CD">0.2</span>)
|
||||
ax.scatter(centroids[:, <span style="color: #B452CD">0</span>], centroids[:, <span style="color: #B452CD">1</span>], c=<span style="color: #CD5555">'black'</span>)
|
||||
|
||||
ax.set_title(<span style="color: #CD5555">"Final Result of K-means Clustering"</span>)
|
||||
|
||||
plt.show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">naive_kmeans</span>(data, n_clusters=<span style="color: #B452CD">4</span>, max_iterations=<span style="color: #B452CD">100</span>, tolerance=<span style="color: #B452CD">1e-8</span>):
|
||||
start_time = time.time()
|
||||
|
||||
n_samples, dimensions = data.shape
|
||||
n_clusters = <span style="color: #B452CD">4</span>
|
||||
<span style="color: #228B22">#np.random.seed(2021)</span>
|
||||
centroids = data[np.random.choice(n_samples, n_clusters, replace=<span style="color: #8B008B; font-weight: bold">False</span>), :]
|
||||
distances = np.zeros((n_samples, n_clusters))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
dist = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> d <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(dimensions):
|
||||
dist += np.abs(data[n, d] - centroids[k, d])**<span style="color: #B452CD">2</span>
|
||||
distances[n, k] = dist
|
||||
|
||||
cluster_labels = np.zeros(n_samples, dtype=<span style="color: #CD5555">'int'</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
smallest = <span style="color: #B452CD">1e10</span>
|
||||
smallest_row_index = <span style="color: #B452CD">1e10</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> distances[n, k] < smallest:
|
||||
smallest = distances[n, k]
|
||||
smallest_row_index = k
|
||||
|
||||
cluster_labels[n] = smallest_row_index
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> iteration <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(max_iterations):
|
||||
prev_centroids = centroids.copy()
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
vector_mean = np.zeros(dimensions)
|
||||
mean_divisor = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> cluster_labels[n] == k:
|
||||
vector_mean += data[n, :]
|
||||
mean_divisor += <span style="color: #B452CD">1</span>
|
||||
|
||||
centroids[k, :] = vector_mean / mean_divisor
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
dist = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> d <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(dimensions):
|
||||
dist += np.abs(data[n, d] - centroids[k, d])**<span style="color: #B452CD">2</span>
|
||||
distances[n, k] = dist
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> n <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_samples):
|
||||
smallest = <span style="color: #B452CD">1e10</span>
|
||||
smallest_row_index = <span style="color: #B452CD">1e10</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_clusters):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> distances[n, k] < smallest:
|
||||
smallest = distances[n, k]
|
||||
smallest_row_index = k
|
||||
|
||||
cluster_labels[n] = smallest_row_index
|
||||
|
||||
centroid_difference = np.sum(np.abs(centroids - prev_centroids))
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> centroid_difference < tolerance:
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f'Converged at iteration {</span>iteration<span style="color: #CD5555">}'</span>)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f'Runtime: {</span>time.time() - start_time<span style="color: #CD5555">} seconds'</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> cluster_labels, centroids
|
||||
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f'Did not converge in {</span>max_iterations<span style="color: #CD5555">} iterations'</span>)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f'Runtime: {</span>time.time() - start_time<span style="color: #CD5555">} seconds'</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> cluster_labels, centroids
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>Decision trees and wrapping up PCA.</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="decision-trees-overarching-aims">Decision trees, overarching aims </h2>
|
||||
|
||||
@@ -8,8 +8,8 @@ doconce format html week44.do.txt --pygments_html_style=default --html_style=blo
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 44: From Decision Trees to Bagging methods">
|
||||
<title>Week 44: From Decision Trees to Bagging methods</title>
|
||||
<meta name="description" content="Week 44: Dimensionality Reduction, PCA and Clustering. Decision Trees">
|
||||
<title>Week 44: Dimensionality Reduction, PCA and Clustering. Decision Trees</title>
|
||||
<style type="text/css">
|
||||
/* bloodish style */
|
||||
body {
|
||||
@@ -141,7 +141,45 @@ div.toc p,a {
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 44', 2, None, 'overview-of-week-44'),
|
||||
('Thursday', 2, None, 'thursday'),
|
||||
('Thursday, Principal Component Analysis',
|
||||
2,
|
||||
None,
|
||||
'thursday-principal-component-analysis'),
|
||||
('Thursday: Clustering and Unsupervised Learning',
|
||||
2,
|
||||
None,
|
||||
'thursday-clustering-and-unsupervised-learning'),
|
||||
('Basic Idea of the $k$-means Clustering Algorithm',
|
||||
2,
|
||||
None,
|
||||
'basic-idea-of-the-k-means-clustering-algorithm'),
|
||||
('The $k$-means Algorithm', 2, None, 'the-k-means-algorithm'),
|
||||
('Basic Math of the $k$-means Algorithm',
|
||||
2,
|
||||
None,
|
||||
'basic-math-of-the-k-means-algorithm'),
|
||||
('Within Cluster Point Scatter',
|
||||
2,
|
||||
None,
|
||||
'within-cluster-point-scatter'),
|
||||
('More Details', 2, None, 'more-details'),
|
||||
('Total Cluster Variance', 2, None, 'total-cluster-variance'),
|
||||
('The $k$-means Clustering Algorithm',
|
||||
2,
|
||||
None,
|
||||
'the-k-means-clustering-algorithm'),
|
||||
('Summarizing', 2, None, 'summarizing'),
|
||||
('Writing our own Code, the Data Set',
|
||||
2,
|
||||
None,
|
||||
'writing-our-own-code-the-data-set'),
|
||||
('Implementing the $k$-means Algorithm',
|
||||
2,
|
||||
None,
|
||||
'implementing-the-k-means-algorithm'),
|
||||
('Plotting', 2, None, 'plotting'),
|
||||
('Continuing', 2, None, 'continuing'),
|
||||
('Wrapping it up', 2, None, 'wrapping-it-up'),
|
||||
('Decision trees, overarching aims',
|
||||
2,
|
||||
None,
|
||||
@@ -297,7 +335,7 @@ MathJax.Hub.Config({
|
||||
|
||||
<!-- ------------------- main content ---------------------- -->
|
||||
<center>
|
||||
<h1>Week 44: From Decision Trees to Bagging methods</h1>
|
||||
<h1>Week 44: Dimensionality Reduction, PCA and Clustering. Decision Trees</h1>
|
||||
</center> <!-- document title -->
|
||||
|
||||
<!-- author(s): Morten Hjorth-Jensen -->
|
||||
@@ -313,7 +351,7 @@ MathJax.Hub.Config({
|
||||
</center>
|
||||
<br>
|
||||
<center>
|
||||
<h4>Nov 1, 2021</h4>
|
||||
<h4>Nov 2, 2021</h4>
|
||||
</center> <!-- date -->
|
||||
<br>
|
||||
|
||||
@@ -321,15 +359,594 @@ MathJax.Hub.Config({
|
||||
<h2 id="overview-of-week-44">Overview of week 44 </h2>
|
||||
|
||||
<ul>
|
||||
<li> Thursday: Wrapping up PCA from last week and basics of decision trees, classification and regression algorithms</li>
|
||||
<li> Thursday: Wrapping up PCA from last week, Clustering and basics of decision trees, classification and regression algorithms</li>
|
||||
<li> Friday: Decision trees, voting models and bagging</li>
|
||||
</ul>
|
||||
<p>Geron's chapter 6 covers decision trees while ensemble models, voting and bagging are discussed in chapter 7. See also lecture from <a href="https://www.uio.no/studier/emner/matnat/math/STK-IN4300/h20/slides/lecture_7.pdf" target="_blank">STK-IN4300, lecture 7</a>. Chapter 9.2 of Hastie et al contains also a good discussion.</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="thursday">Thursday </h2>
|
||||
<h2 id="thursday-principal-component-analysis">Thursday, Principal Component Analysis </h2>
|
||||
|
||||
<p>For the principal component analysis,
|
||||
see slides from <a href="https://compphysics.github.io/MachineLearning/doc/pub/week43/html/week43-reveal.html" target="_blank">week 43</a>, in particular from slide 28 and forward
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="thursday-clustering-and-unsupervised-learning">Thursday: Clustering and Unsupervised Learning </h2>
|
||||
|
||||
<p>In general terms cluster analysis, or clustering, is the task of grouping a
|
||||
data-set into different distinct categories based on some measure of equality of
|
||||
the data. This measure is often referred to as a <b>metric</b> or <b>similarity
|
||||
measure</b> in the literature (note: sometimes we deal with a <b>dissimilarity
|
||||
measure</b> instead). Usually, these metrics are formulated as some kind of
|
||||
distance function between points in a high-dimensional space.
|
||||
</p>
|
||||
|
||||
<p>The simplest, and also the most
|
||||
common is the <b>Euclidean distance</b>.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="basic-idea-of-the-k-means-clustering-algorithm">Basic Idea of the \( k \)-means Clustering Algorithm </h2>
|
||||
|
||||
<p>The simplest of all clustering algorithms is the <b>k-means algorithm</b>
|
||||
, sometimes also referred to as <em>Lloyds algorithm</em>. It is the simplest and also
|
||||
the most common. From its simplicity it obtains both strengths and weaknesses.
|
||||
These will be discussed in more detail later. The \( k \)-means algorithm is a
|
||||
<b>centroid based</b> clustering algorithm.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="the-k-means-algorithm">The \( k \)-means Algorithm </h2>
|
||||
|
||||
<p>Assume, we are given \( n \) data points and we wish to split the data into \( K < n \)
|
||||
different categories, or clusters. We label each cluster by an integer
|
||||
</p>
|
||||
|
||||
$$ k\in\{1, \cdots, K \}$.
|
||||
$$
|
||||
|
||||
<p>In the basic k-means algorithm each point is assigned to only
|
||||
one cluster \( k \), and these assignments are <em>non-injective</em> i.e. many-to-one. We
|
||||
can think of these mappings as an encoder \( k = C(i) \), which assigns the \( i \)-th
|
||||
data-point \( \bf x_i \) to the \( k \)-th cluster.
|
||||
</p>
|
||||
|
||||
<p>\( k \)-means algorithm in words:</p>
|
||||
<ol>
|
||||
<li> We start with guesses / random initializations of our \( k \) cluster centers / centroids</li>
|
||||
<li> For each centroid the points that are most similar are identified</li>
|
||||
<li> Then we move / replace each centroid with a coordinate average of all the points that were assigned to that centroid.</li>
|
||||
<li> Iterate this points 2, 3) until the centroids no longer move (to some tolerance)</li>
|
||||
</ol>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="basic-math-of-the-k-means-algorithm">Basic Math of the \( k \)-means Algorithm </h2>
|
||||
|
||||
<p>We assume we have \( n \) data-points</p>
|
||||
$$
|
||||
\begin{equation}\label{eq:kmeanspoints}
|
||||
\boldsymbol{x_i} = \{x_{i, 1}, \cdots, x_{i, p}\}\in\mathbb{R}^p.
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>which we wish to group into \( K < n \) clusters. For our dissimilarity measure we
|
||||
use the <em>squared Euclidean distance</em>
|
||||
</p>
|
||||
$$
|
||||
\begin{equation}\label{eq:squaredeuclidean}
|
||||
d(\boldsymbol{x_i}, \boldsymbol{x_i'}) = \sum_{j=1}^p(x_{ij} - x_{i'j})^2
|
||||
= ||\boldsymbol{x_i} - \boldsymbol{x_{i'}}||^2
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="within-cluster-point-scatter">Within Cluster Point Scatter </h2>
|
||||
|
||||
<p>We define the so called <em>within-cluster point scatter</em> which gives us a
|
||||
measure of how close each data point assigned to the same cluster tends to be to
|
||||
the all the others.
|
||||
</p>
|
||||
$$
|
||||
\begin{equation}\label{eq:withincluster}
|
||||
W(C) = \frac{1}{2}\sum_{k=1}^K\sum_{C(i)=k}
|
||||
\sum_{C(i')=k}d(\boldsymbol{x_i}, \boldsymbol{x_{i'}}) =
|
||||
\sum_{k=1}^KN_k\sum_{C(i)=k}||\boldsymbol{x_i} - \boldsymbol{\overline{x_k}}||^2
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>where \( \boldsymbol{\overline{x_k}} \) is the mean vector associated with the \( k \)-th
|
||||
cluster, and \( N_k = \sum_{i=1}^nI(C(i) = k) \), where the \( I() \) notation is
|
||||
similar to the Kronecker delta (<em>Commonly used in statistics, it just means that
|
||||
when \( i = k \) we have the encoder \( C(i) \)</em>). In other words, the within-cluster
|
||||
scatter measures the compactness of each cluster with respect to the data points
|
||||
assigned to each cluster. This is the quantity that the \( k \)-means algorithm aims
|
||||
to minimize. We refer to this quantity \( W(C) \) as the within cluster scatter
|
||||
because of its relation to the <em>total scatter</em>.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="more-details">More Details </h2>
|
||||
|
||||
<p>We have</p>
|
||||
$$
|
||||
\begin{equation}\label{eq:totalscatter}
|
||||
T = W(C) + B(C) = \frac{1}{2}\sum_{i=1}^n
|
||||
\sum_{i'=1}^nd(\boldsymbol{x_i}, \boldsymbol{x_{i'}})
|
||||
= \frac{1}{2}\sum_{k=1}^K\sum_{C(i)=k}
|
||||
\Big(\sum_{C(i') = k}d(\boldsymbol{x_i}, \boldsymbol{x_{i'}})
|
||||
+ \sum_{C(i')\neq k}d(\boldsymbol{x_i}, \boldsymbol{x_{i'}})\Big).
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>This is a quantity that is conserved throughout the \( k \)-means algorithm. It can
|
||||
be thought of as the total amount of information in the data, and it is composed
|
||||
of the aforementioned within-cluster scatter and the <em>between-cluster scatter</em>
|
||||
\( B(C) \). In methods such as principle component analysis the total scatter is not
|
||||
conserved.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="total-cluster-variance">Total Cluster Variance </h2>
|
||||
<p>Given a cluster mean \( \boldsymbol{m_k} \) we define the <b>total cluster variance</b></p>
|
||||
$$
|
||||
\begin{equation}\label{eq:totalclustervariance}
|
||||
\min_{C, \{\boldsymbol{m_k}\}_1^K}\sum_{k=1}^KN_k\sum||\boldsymbol{x_i} - \boldsymbol{m_k}||^2
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>Now we have all the pieces necessary to formally revisit the \( k \)-means algorithm.</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="the-k-means-clustering-algorithm">The \( k \)-means Clustering Algorithm </h2>
|
||||
|
||||
<p>The \( k \)-means clustering algorithm goes as follows </p>
|
||||
|
||||
<ol>
|
||||
<li> For a given cluster assignment \( C \), and \( k \) cluster means \( \left{m_1, \cdots, m_k\right} \). We minimize the total cluster variance with respect to the cluster means \( \{m_k\} \) yielding the means of the currently assigned clusters.</li>
|
||||
<li> Given a current set of \( k \) means \( \{m_k\} \) the total cluster variance is minimized by assigning each observation to the closest (current) cluster mean. That is $$C(i) = \underset{1\leq k\leq K}{\mathrm{argmin}} ||\boldsymbol{x_i} - \boldsymbol{m_k}||^2$$</li>
|
||||
<li> Steps 1 and 2 are repeated until the assignments do not change.</li>
|
||||
</ol>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="summarizing">Summarizing </h2>
|
||||
|
||||
<ol>
|
||||
<li> Before we start we specify a number \( k \) which is the number of clusters we want to try to separate our data into.</li>
|
||||
<li> We initially choose \( k \) random data points in our data as our initial centroids, <em>or means</em> (this is where the name comes from).</li>
|
||||
<li> Assign each data point to their closest centroid, based on the squared Euclidean distance.</li>
|
||||
<li> For each of the \( k \) cluster we update the centroid by calculating new mean values for all the data points in the cluster.</li>
|
||||
<li> Iteratively minimize the within cluster scatter by performing steps (3, 4) until the new assignments stop changing (can be to some tolerance) or until a maximum number of iterations have passed.</li>
|
||||
</ol>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="writing-our-own-code-the-data-set">Writing our own Code, the Data Set </h2>
|
||||
|
||||
<p>Let us now program the most basic version of the algorithm using nothing but
|
||||
Python with numpy arrays. This code is kept intentionally simple to gradually
|
||||
progress our understanding. There is no vectorization of any kind, and even most
|
||||
helper functions are not utilized.
|
||||
</p>
|
||||
|
||||
<p>We need first a dataset to do our cluster analysis on. In our case
|
||||
this is a plain <em>vanilla</em> data set using random numbers using a
|
||||
Gaussian distribution.
|
||||
</p>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">time</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">tensorflow</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">tf</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">matplotlib</span> <span style="color: #008000; font-weight: bold">import</span> image
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.cluster</span> <span style="color: #008000; font-weight: bold">import</span> KMeans
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> display
|
||||
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">2021</span>)
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>Next we define functions, for ease of use later, to generate Gaussians and to
|
||||
set up our toy data set.
|
||||
</p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">gaussian_points</span>(dim<span style="color: #666666">=2</span>, n_points<span style="color: #666666">=1000</span>, mean_vector<span style="color: #666666">=</span>np<span style="color: #666666">.</span>array([<span style="color: #666666">0</span>, <span style="color: #666666">0</span>]),
|
||||
sample_variance<span style="color: #666666">=1</span>):
|
||||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Very simple custom function to generate gaussian distributed point clusters</span>
|
||||
<span style="color: #BA2121; font-style: italic"> with variable dimension, number of points, means in each direction</span>
|
||||
<span style="color: #BA2121; font-style: italic"> (must match dim) and sample variance.</span>
|
||||
|
||||
<span style="color: #BA2121; font-style: italic"> Inputs:</span>
|
||||
<span style="color: #BA2121; font-style: italic"> dim (int)</span>
|
||||
<span style="color: #BA2121; font-style: italic"> n_points (int)</span>
|
||||
<span style="color: #BA2121; font-style: italic"> mean_vector (np.array) (where index 0 is x, index 1 is y etc.)</span>
|
||||
<span style="color: #BA2121; font-style: italic"> sample_variance (float)</span>
|
||||
|
||||
<span style="color: #BA2121; font-style: italic"> Returns:</span>
|
||||
<span style="color: #BA2121; font-style: italic"> data (np.array): with dimensions (dim x n_points)</span>
|
||||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||||
|
||||
mean_matrix <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(dim) <span style="color: #666666">+</span> mean_vector
|
||||
covariance_matrix <span style="color: #666666">=</span> np<span style="color: #666666">.</span>eye(dim) <span style="color: #666666">*</span> sample_variance
|
||||
data <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>multivariate_normal(mean_matrix, covariance_matrix,
|
||||
n_points)
|
||||
<span style="color: #008000; font-weight: bold">return</span> data
|
||||
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">generate_simple_clustering_dataset</span>(dim<span style="color: #666666">=2</span>, n_points<span style="color: #666666">=1000</span>, plotting<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>,
|
||||
return_data<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>):
|
||||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Toy model to illustrate k-means clustering</span>
|
||||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||||
|
||||
data1 <span style="color: #666666">=</span> gaussian_points(mean_vector<span style="color: #666666">=</span>np<span style="color: #666666">.</span>array([<span style="color: #666666">5</span>, <span style="color: #666666">5</span>]))
|
||||
data2 <span style="color: #666666">=</span> gaussian_points()
|
||||
data3 <span style="color: #666666">=</span> gaussian_points(mean_vector<span style="color: #666666">=</span>np<span style="color: #666666">.</span>array([<span style="color: #666666">1</span>, <span style="color: #666666">4.5</span>]))
|
||||
data4 <span style="color: #666666">=</span> gaussian_points(mean_vector<span style="color: #666666">=</span>np<span style="color: #666666">.</span>array([<span style="color: #666666">5</span>, <span style="color: #666666">1</span>]))
|
||||
data <span style="color: #666666">=</span> np<span style="color: #666666">.</span>concatenate((data1, data2, data3, data4), axis<span style="color: #666666">=0</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> plotting:
|
||||
fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots()
|
||||
ax<span style="color: #666666">.</span>scatter(data[:, <span style="color: #666666">0</span>], data[:, <span style="color: #666666">1</span>], alpha<span style="color: #666666">=0.2</span>)
|
||||
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">'Toy Model Dataset'</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> return_data:
|
||||
<span style="color: #008000; font-weight: bold">return</span> data
|
||||
|
||||
|
||||
data <span style="color: #666666">=</span> generate_simple_clustering_dataset()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="implementing-the-k-means-algorithm">Implementing the \( k \)-means Algorithm </h2>
|
||||
|
||||
<p>With the above dataset we start
|
||||
implementing the \( k \)-means algorithm.
|
||||
</p>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;">n_samples, dimensions <span style="color: #666666">=</span> data<span style="color: #666666">.</span>shape
|
||||
n_clusters <span style="color: #666666">=</span> <span style="color: #666666">4</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># we randomly initialize our centroids</span>
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">2021</span>)
|
||||
centroids <span style="color: #666666">=</span> data[np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>choice(n_samples, n_clusters, replace<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>), :]
|
||||
distances <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((n_samples, n_clusters))
|
||||
|
||||
<span style="color: #408080; font-style: italic"># first we need to calculate the distance to each centroid from our data</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> k <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_clusters):
|
||||
<span style="color: #008000; font-weight: bold">for</span> n <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_samples):
|
||||
dist <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> d <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(dimensions):
|
||||
dist <span style="color: #666666">+=</span> np<span style="color: #666666">.</span>abs(data[n, d] <span style="color: #666666">-</span> centroids[k, d])<span style="color: #666666">**2</span>
|
||||
distances[n, k] <span style="color: #666666">=</span> dist
|
||||
|
||||
<span style="color: #408080; font-style: italic"># we initialize an array to keep track of to which cluster each point belongs</span>
|
||||
<span style="color: #408080; font-style: italic"># the way we set it up here the index tracks which point and the value which</span>
|
||||
<span style="color: #408080; font-style: italic"># cluster the point belongs to</span>
|
||||
cluster_labels <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(n_samples, dtype<span style="color: #666666">=</span><span style="color: #BA2121">'int'</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># next we loop through our samples and for every point assign it to the cluster</span>
|
||||
<span style="color: #408080; font-style: italic"># to which it has the smallest distance to</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> n <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_samples):
|
||||
<span style="color: #408080; font-style: italic"># tracking variables (all of this is basically just an argmin)</span>
|
||||
smallest <span style="color: #666666">=</span> <span style="color: #666666">1e10</span>
|
||||
smallest_row_index <span style="color: #666666">=</span> <span style="color: #666666">1e10</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> k <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_clusters):
|
||||
<span style="color: #008000; font-weight: bold">if</span> distances[n, k] <span style="color: #666666"><</span> smallest:
|
||||
smallest <span style="color: #666666">=</span> distances[n, k]
|
||||
smallest_row_index <span style="color: #666666">=</span> k
|
||||
|
||||
cluster_labels[n] <span style="color: #666666">=</span> smallest_row_index
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="plotting">Plotting </h2>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;">fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
|
||||
ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot()
|
||||
unique_cluster_labels <span style="color: #666666">=</span> np<span style="color: #666666">.</span>unique(cluster_labels)
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> unique_cluster_labels:
|
||||
ax<span style="color: #666666">.</span>scatter(data[cluster_labels <span style="color: #666666">==</span> i, <span style="color: #666666">0</span>],
|
||||
data[cluster_labels <span style="color: #666666">==</span> i, <span style="color: #666666">1</span>],
|
||||
label <span style="color: #666666">=</span> i,
|
||||
alpha <span style="color: #666666">=</span> <span style="color: #666666">0.2</span>)
|
||||
ax<span style="color: #666666">.</span>scatter(centroids[:, <span style="color: #666666">0</span>], centroids[:, <span style="color: #666666">1</span>], c<span style="color: #666666">=</span><span style="color: #BA2121">'black'</span>)
|
||||
|
||||
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">"First Grouping of Points to Centroids"</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>So what do we have so far? We have 'picked' \( k \) centroids at random from our
|
||||
data points. There are other ways of more intelligently choosing their
|
||||
initializations, however for our purposes randomly is fine. Then we have
|
||||
initialized an array 'distances' which holds the information of the distance,
|
||||
<em>or dissimilarity</em>, of every point to of our centroids. Finally, we have
|
||||
initialized an array 'cluster_labels' which according to our distances array
|
||||
holds the information of to which centroid every point is assigned. This was the
|
||||
first pass of our algorithm. Essentially, all we need to do now is repeat the
|
||||
distance and assignment steps above until we have reached a desired convergence
|
||||
or a maximum amount of iterations.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="continuing">Continuing </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;">max_iterations <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
tolerance <span style="color: #666666">=</span> <span style="color: #666666">1e-8</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> iteration <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(max_iterations):
|
||||
prev_centroids <span style="color: #666666">=</span> centroids<span style="color: #666666">.</span>copy()
|
||||
<span style="color: #008000; font-weight: bold">for</span> k <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_clusters):
|
||||
<span style="color: #408080; font-style: italic"># this array will be used to update our centroid positions</span>
|
||||
vector_mean <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(dimensions)
|
||||
mean_divisor <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> n <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_samples):
|
||||
<span style="color: #008000; font-weight: bold">if</span> cluster_labels[n] <span style="color: #666666">==</span> k:
|
||||
vector_mean <span style="color: #666666">+=</span> data[n, :]
|
||||
mean_divisor <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># update according to the k means</span>
|
||||
centroids[k, :] <span style="color: #666666">=</span> vector_mean <span style="color: #666666">/</span> mean_divisor
|
||||
|
||||
<span style="color: #408080; font-style: italic"># we find the dissimilarity</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> k <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_clusters):
|
||||
<span style="color: #008000; font-weight: bold">for</span> n <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_samples):
|
||||
dist <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> d <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(dimensions):
|
||||
dist <span style="color: #666666">+=</span> np<span style="color: #666666">.</span>abs(data[n, d] <span style="color: #666666">-</span> centroids[k, d])<span style="color: #666666">**2</span>
|
||||
distances[n, k] <span style="color: #666666">=</span> dist
|
||||
|
||||
<span style="color: #408080; font-style: italic"># assign each point</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> n <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_samples):
|
||||
smallest <span style="color: #666666">=</span> <span style="color: #666666">1e10</span>
|
||||
smallest_row_index <span style="color: #666666">=</span> <span style="color: #666666">1e10</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> k <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_clusters):
|
||||
<span style="color: #008000; font-weight: bold">if</span> distances[n, k] <span style="color: #666666"><</span> smallest:
|
||||
smallest <span style="color: #666666">=</span> distances[n, k]
|
||||
smallest_row_index <span style="color: #666666">=</span> k
|
||||
|
||||
cluster_labels[n] <span style="color: #666666">=</span> smallest_row_index
|
||||
|
||||
<span style="color: #408080; font-style: italic"># convergence criteria</span>
|
||||
centroid_difference <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(np<span style="color: #666666">.</span>abs(centroids <span style="color: #666666">-</span> prev_centroids))
|
||||
<span style="color: #008000; font-weight: bold">if</span> centroid_difference <span style="color: #666666"><</span> tolerance:
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f'Converged at iteration </span><span style="color: #BB6688; font-weight: bold">{</span>iteration<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">'</span>)
|
||||
<span style="color: #008000; font-weight: bold">break</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">elif</span> iteration <span style="color: #666666">==</span> max_iterations:
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f'Did not converge in </span><span style="color: #BB6688; font-weight: bold">{</span>max_iterations<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121"> iterations'</span>)
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="wrapping-it-up">Wrapping it up </h2>
|
||||
<p>We now have a simple , un-optimized \( k \)-means
|
||||
clustering implementation. Lets plot the final result
|
||||
</p>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;">fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
|
||||
ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot()
|
||||
unique_cluster_labels <span style="color: #666666">=</span> np<span style="color: #666666">.</span>unique(cluster_labels)
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> unique_cluster_labels:
|
||||
ax<span style="color: #666666">.</span>scatter(data[cluster_labels <span style="color: #666666">==</span> i, <span style="color: #666666">0</span>],
|
||||
data[cluster_labels <span style="color: #666666">==</span> i, <span style="color: #666666">1</span>],
|
||||
label <span style="color: #666666">=</span> i,
|
||||
alpha <span style="color: #666666">=</span> <span style="color: #666666">0.2</span>)
|
||||
ax<span style="color: #666666">.</span>scatter(centroids[:, <span style="color: #666666">0</span>], centroids[:, <span style="color: #666666">1</span>], c<span style="color: #666666">=</span><span style="color: #BA2121">'black'</span>)
|
||||
|
||||
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">"Final Result of K-means Clustering"</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">naive_kmeans</span>(data, n_clusters<span style="color: #666666">=4</span>, max_iterations<span style="color: #666666">=100</span>, tolerance<span style="color: #666666">=1e-8</span>):
|
||||
start_time <span style="color: #666666">=</span> time<span style="color: #666666">.</span>time()
|
||||
|
||||
n_samples, dimensions <span style="color: #666666">=</span> data<span style="color: #666666">.</span>shape
|
||||
n_clusters <span style="color: #666666">=</span> <span style="color: #666666">4</span>
|
||||
<span style="color: #408080; font-style: italic">#np.random.seed(2021)</span>
|
||||
centroids <span style="color: #666666">=</span> data[np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>choice(n_samples, n_clusters, replace<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>), :]
|
||||
distances <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((n_samples, n_clusters))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> k <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_clusters):
|
||||
<span style="color: #008000; font-weight: bold">for</span> n <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_samples):
|
||||
dist <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> d <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(dimensions):
|
||||
dist <span style="color: #666666">+=</span> np<span style="color: #666666">.</span>abs(data[n, d] <span style="color: #666666">-</span> centroids[k, d])<span style="color: #666666">**2</span>
|
||||
distances[n, k] <span style="color: #666666">=</span> dist
|
||||
|
||||
cluster_labels <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(n_samples, dtype<span style="color: #666666">=</span><span style="color: #BA2121">'int'</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> n <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_samples):
|
||||
smallest <span style="color: #666666">=</span> <span style="color: #666666">1e10</span>
|
||||
smallest_row_index <span style="color: #666666">=</span> <span style="color: #666666">1e10</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> k <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_clusters):
|
||||
<span style="color: #008000; font-weight: bold">if</span> distances[n, k] <span style="color: #666666"><</span> smallest:
|
||||
smallest <span style="color: #666666">=</span> distances[n, k]
|
||||
smallest_row_index <span style="color: #666666">=</span> k
|
||||
|
||||
cluster_labels[n] <span style="color: #666666">=</span> smallest_row_index
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> iteration <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(max_iterations):
|
||||
prev_centroids <span style="color: #666666">=</span> centroids<span style="color: #666666">.</span>copy()
|
||||
<span style="color: #008000; font-weight: bold">for</span> k <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_clusters):
|
||||
vector_mean <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(dimensions)
|
||||
mean_divisor <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> n <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_samples):
|
||||
<span style="color: #008000; font-weight: bold">if</span> cluster_labels[n] <span style="color: #666666">==</span> k:
|
||||
vector_mean <span style="color: #666666">+=</span> data[n, :]
|
||||
mean_divisor <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
||||
|
||||
centroids[k, :] <span style="color: #666666">=</span> vector_mean <span style="color: #666666">/</span> mean_divisor
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> k <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_clusters):
|
||||
<span style="color: #008000; font-weight: bold">for</span> n <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_samples):
|
||||
dist <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> d <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(dimensions):
|
||||
dist <span style="color: #666666">+=</span> np<span style="color: #666666">.</span>abs(data[n, d] <span style="color: #666666">-</span> centroids[k, d])<span style="color: #666666">**2</span>
|
||||
distances[n, k] <span style="color: #666666">=</span> dist
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> n <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_samples):
|
||||
smallest <span style="color: #666666">=</span> <span style="color: #666666">1e10</span>
|
||||
smallest_row_index <span style="color: #666666">=</span> <span style="color: #666666">1e10</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> k <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_clusters):
|
||||
<span style="color: #008000; font-weight: bold">if</span> distances[n, k] <span style="color: #666666"><</span> smallest:
|
||||
smallest <span style="color: #666666">=</span> distances[n, k]
|
||||
smallest_row_index <span style="color: #666666">=</span> k
|
||||
|
||||
cluster_labels[n] <span style="color: #666666">=</span> smallest_row_index
|
||||
|
||||
centroid_difference <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(np<span style="color: #666666">.</span>abs(centroids <span style="color: #666666">-</span> prev_centroids))
|
||||
<span style="color: #008000; font-weight: bold">if</span> centroid_difference <span style="color: #666666"><</span> tolerance:
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f'Converged at iteration </span><span style="color: #BB6688; font-weight: bold">{</span>iteration<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">'</span>)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f'Runtime: </span><span style="color: #BB6688; font-weight: bold">{</span>time<span style="color: #666666">.</span>time() <span style="color: #666666">-</span> start_time<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121"> seconds'</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> cluster_labels, centroids
|
||||
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f'Did not converge in </span><span style="color: #BB6688; font-weight: bold">{</span>max_iterations<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121"> iterations'</span>)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f'Runtime: </span><span style="color: #BB6688; font-weight: bold">{</span>time<span style="color: #666666">.</span>time() <span style="color: #666666">-</span> start_time<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121"> seconds'</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> cluster_labels, centroids
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>Decision trees and wrapping up PCA.</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="decision-trees-overarching-aims">Decision trees, overarching aims </h2>
|
||||
|
||||
Binary file not shown.
+855
-131
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,4 @@
|
||||
TITLE: Week 44: From Decision Trees to Bagging methods
|
||||
TITLE: Week 44: Dimensionality Reduction, PCA and Clustering. Decision Trees
|
||||
AUTHOR: Morten Hjorth-Jensen {copyright, 1999-present|CC BY-NC} at Department of Physics, University of Oslo & Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
|
||||
DATE: today
|
||||
|
||||
@@ -6,7 +6,7 @@ DATE: today
|
||||
!split
|
||||
===== Overview of week 44 =====
|
||||
|
||||
* Thursday: Wrapping up PCA from last week and basics of decision trees, classification and regression algorithms
|
||||
* Thursday: Wrapping up PCA from last week, Clustering and basics of decision trees, classification and regression algorithms
|
||||
* Friday: Decision trees, voting models and bagging
|
||||
|
||||
|
||||
@@ -14,9 +14,460 @@ Geron's chapter 6 covers decision trees while ensemble models, voting and baggin
|
||||
|
||||
|
||||
!split
|
||||
===== Thursday =====
|
||||
===== Thursday, Principal Component Analysis =====
|
||||
|
||||
For the principal component analysis,
|
||||
see slides from "week 43":"https://compphysics.github.io/MachineLearning/doc/pub/week43/html/week43-reveal.html", in particular from slide 28 and forward
|
||||
|
||||
!split
|
||||
===== Thursday: Clustering and Unsupervised Learning =====
|
||||
|
||||
In general terms cluster analysis, or clustering, is the task of grouping a
|
||||
data-set into different distinct categories based on some measure of equality of
|
||||
the data. This measure is often referred to as a _metric_ or _similarity
|
||||
measure_ in the literature (note: sometimes we deal with a _dissimilarity
|
||||
measure_ instead). Usually, these metrics are formulated as some kind of
|
||||
distance function between points in a high-dimensional space.
|
||||
|
||||
The simplest, and also the most
|
||||
common is the _Euclidean distance_.
|
||||
|
||||
!split
|
||||
===== Basic Idea of the $k$-means Clustering Algorithm =====
|
||||
|
||||
The simplest of all clustering algorithms is the _k-means algorithm_
|
||||
, sometimes also referred to as *Lloyds algorithm*. It is the simplest and also
|
||||
the most common. From its simplicity it obtains both strengths and weaknesses.
|
||||
These will be discussed in more detail later. The $k$-means algorithm is a
|
||||
_centroid based_ clustering algorithm.
|
||||
|
||||
!split
|
||||
===== The $k$-means Algorithm =====
|
||||
|
||||
Assume, we are given $n$ data points and we wish to split the data into $K < n$
|
||||
different categories, or clusters. We label each cluster by an integer
|
||||
|
||||
!bt
|
||||
\[ k\in\{1, \cdots, K \}$.
|
||||
\]
|
||||
!et
|
||||
|
||||
In the basic k-means algorithm each point is assigned to only
|
||||
one cluster $k$, and these assignments are *non-injective* i.e. many-to-one. We
|
||||
can think of these mappings as an encoder $k = C(i)$, which assigns the $i$-th
|
||||
data-point $\bf x_i$ to the $k$-th cluster.
|
||||
|
||||
$k$-means algorithm in words:
|
||||
o We start with guesses / random initializations of our $k$ cluster centers /
|
||||
centroids
|
||||
o For each centroid the points that are most similar are identified
|
||||
o Then we move / replace each centroid with a coordinate average of all the
|
||||
points that were assigned to that centroid.
|
||||
o Iterate this points 2, 3) until the centroids no longer move (to some
|
||||
tolerance)
|
||||
|
||||
!split
|
||||
===== Basic Math of the $k$-means Algorithm =====
|
||||
|
||||
We assume we have $n$ data-points
|
||||
!bt
|
||||
\begin{equation}\label{eq:kmeanspoints}
|
||||
\bm{x_i} = \{x_{i, 1}, \cdots, x_{i, p}\}\in\mathbb{R}^p.
|
||||
\end{equation}
|
||||
!et
|
||||
which we wish to group into $K < n$ clusters. For our dissimilarity measure we
|
||||
use the *squared Euclidean distance*
|
||||
!bt
|
||||
\begin{equation}\label{eq:squaredeuclidean}
|
||||
d(\bm{x_i}, \bm{x_i'}) = \sum_{j=1}^p(x_{ij} - x_{i'j})^2
|
||||
= ||\bm{x_i} - \bm{x_{i'}}||^2
|
||||
\end{equation}
|
||||
!et
|
||||
|
||||
!split
|
||||
===== Within Cluster Point Scatter =====
|
||||
|
||||
We define the so called *within-cluster point scatter* which gives us a
|
||||
measure of how close each data point assigned to the same cluster tends to be to
|
||||
the all the others.
|
||||
!bt
|
||||
\begin{equation}\label{eq:withincluster}
|
||||
W(C) = \frac{1}{2}\sum_{k=1}^K\sum_{C(i)=k}
|
||||
\sum_{C(i')=k}d(\bm{x_i}, \bm{x_{i'}}) =
|
||||
\sum_{k=1}^KN_k\sum_{C(i)=k}||\bm{x_i} - \bm{\overline{x_k}}||^2
|
||||
\end{equation}
|
||||
!et
|
||||
where $\bm{\overline{x_k}}$ is the mean vector associated with the $k$-th
|
||||
cluster, and $N_k = \sum_{i=1}^nI(C(i) = k)$, where the $I()$ notation is
|
||||
similar to the Kronecker delta (*Commonly used in statistics, it just means that
|
||||
when $i = k$ we have the encoder $C(i)$*). In other words, the within-cluster
|
||||
scatter measures the compactness of each cluster with respect to the data points
|
||||
assigned to each cluster. This is the quantity that the $k$-means algorithm aims
|
||||
to minimize. We refer to this quantity $W(C)$ as the within cluster scatter
|
||||
because of its relation to the *total scatter*.
|
||||
|
||||
|
||||
!split
|
||||
===== More Details =====
|
||||
|
||||
We have
|
||||
!bt
|
||||
\begin{equation}\label{eq:totalscatter}
|
||||
T = W(C) + B(C) = \frac{1}{2}\sum_{i=1}^n
|
||||
\sum_{i'=1}^nd(\bm{x_i}, \bm{x_{i'}})
|
||||
= \frac{1}{2}\sum_{k=1}^K\sum_{C(i)=k}
|
||||
\Big(\sum_{C(i') = k}d(\bm{x_i}, \bm{x_{i'}})
|
||||
+ \sum_{C(i')\neq k}d(\bm{x_i}, \bm{x_{i'}})\Big).
|
||||
\end{equation}
|
||||
!et
|
||||
|
||||
This is a quantity that is conserved throughout the $k$-means algorithm. It can
|
||||
be thought of as the total amount of information in the data, and it is composed
|
||||
of the aforementioned within-cluster scatter and the *between-cluster scatter*
|
||||
$B(C)$. In methods such as principle component analysis the total scatter is not
|
||||
conserved.
|
||||
|
||||
!split
|
||||
===== Total Cluster Variance =====
|
||||
Given a cluster mean $\bm{m_k}$ we define the _total cluster variance_
|
||||
!bt
|
||||
\begin{equation}\label{eq:totalclustervariance}
|
||||
\min_{C, \{\bm{m_k}\}_1^K}\sum_{k=1}^KN_k\sum||\bm{x_i} - \bm{m_k}||^2
|
||||
\end{equation}
|
||||
!et
|
||||
Now we have all the pieces necessary to formally revisit the $k$-means algorithm.
|
||||
|
||||
|
||||
!split
|
||||
===== The $k$-means Clustering Algorithm =====
|
||||
|
||||
The $k$-means clustering algorithm goes as follows
|
||||
|
||||
o For a given cluster assignment $C$, and $k$ cluster means
|
||||
$\left{m_1, \cdots, m_k\right}$. We minimize the total cluster variance with respect to
|
||||
the cluster means $\{m_k\}$ yielding the means of the currently assigned
|
||||
clusters.
|
||||
o Given a current set of $k$ means $\{m_k\}$ the total cluster variance is
|
||||
minimized by assigning each observation to the closest (current) cluster mean.
|
||||
That is $$C(i) = \underset{1\leq k\leq K}{\mathrm{argmin}}
|
||||
||\bm{x_i} - \bm{m_k}||^2$$
|
||||
o Steps 1 and 2 are repeated until the assignments do not change.
|
||||
|
||||
|
||||
!split
|
||||
===== Summarizing =====
|
||||
|
||||
|
||||
o Before we start we specify a number $k$ which is the number of clusters we
|
||||
want to try to separate our data into.
|
||||
o We initially choose $k$ random data points in our data as our initial
|
||||
centroids, *or means* (this is where the name comes from).
|
||||
o Assign each data point to their closest centroid, based on the squared
|
||||
Euclidean distance.
|
||||
o For each of the $k$ cluster we update the centroid by calculating new mean
|
||||
values for all the data points in the cluster.
|
||||
o Iteratively minimize the within cluster scatter by performing steps (3, 4)
|
||||
until the new assignments stop changing (can be to some tolerance) or
|
||||
until a maximum number of iterations have passed.
|
||||
|
||||
|
||||
!split
|
||||
===== Writing our own Code, the Data Set =====
|
||||
|
||||
Let us now program the most basic version of the algorithm using nothing but
|
||||
Python with numpy arrays. This code is kept intentionally simple to gradually
|
||||
progress our understanding. There is no vectorization of any kind, and even most
|
||||
helper functions are not utilized.
|
||||
|
||||
We need first a dataset to do our cluster analysis on. In our case
|
||||
this is a plain *vanilla* data set using random numbers using a
|
||||
Gaussian distribution.
|
||||
|
||||
!bc pycod
|
||||
import time
|
||||
import numpy as np
|
||||
import tensorflow as tf
|
||||
from matplotlib import image
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.cluster import KMeans
|
||||
from IPython.display import display
|
||||
|
||||
np.random.seed(2021)
|
||||
|
||||
!ec
|
||||
|
||||
Next we define functions, for ease of use later, to generate Gaussians and to
|
||||
set up our toy data set.
|
||||
!bc pycod
|
||||
def gaussian_points(dim=2, n_points=1000, mean_vector=np.array([0, 0]),
|
||||
sample_variance=1):
|
||||
"""
|
||||
Very simple custom function to generate gaussian distributed point clusters
|
||||
with variable dimension, number of points, means in each direction
|
||||
(must match dim) and sample variance.
|
||||
|
||||
Inputs:
|
||||
dim (int)
|
||||
n_points (int)
|
||||
mean_vector (np.array) (where index 0 is x, index 1 is y etc.)
|
||||
sample_variance (float)
|
||||
|
||||
Returns:
|
||||
data (np.array): with dimensions (dim x n_points)
|
||||
"""
|
||||
|
||||
mean_matrix = np.zeros(dim) + mean_vector
|
||||
covariance_matrix = np.eye(dim) * sample_variance
|
||||
data = np.random.multivariate_normal(mean_matrix, covariance_matrix,
|
||||
n_points)
|
||||
return data
|
||||
|
||||
|
||||
|
||||
def generate_simple_clustering_dataset(dim=2, n_points=1000, plotting=True,
|
||||
return_data=True):
|
||||
"""
|
||||
Toy model to illustrate k-means clustering
|
||||
"""
|
||||
|
||||
data1 = gaussian_points(mean_vector=np.array([5, 5]))
|
||||
data2 = gaussian_points()
|
||||
data3 = gaussian_points(mean_vector=np.array([1, 4.5]))
|
||||
data4 = gaussian_points(mean_vector=np.array([5, 1]))
|
||||
data = np.concatenate((data1, data2, data3, data4), axis=0)
|
||||
|
||||
if plotting:
|
||||
fig, ax = plt.subplots()
|
||||
ax.scatter(data[:, 0], data[:, 1], alpha=0.2)
|
||||
ax.set_title('Toy Model Dataset')
|
||||
plt.show()
|
||||
|
||||
|
||||
if return_data:
|
||||
return data
|
||||
|
||||
|
||||
data = generate_simple_clustering_dataset()
|
||||
!ec
|
||||
|
||||
|
||||
!split
|
||||
===== Implementing the $k$-means Algorithm =====
|
||||
|
||||
With the above dataset we start
|
||||
implementing the $k$-means algorithm.
|
||||
|
||||
!bc pycod
|
||||
|
||||
n_samples, dimensions = data.shape
|
||||
n_clusters = 4
|
||||
|
||||
# we randomly initialize our centroids
|
||||
np.random.seed(2021)
|
||||
centroids = data[np.random.choice(n_samples, n_clusters, replace=False), :]
|
||||
distances = np.zeros((n_samples, n_clusters))
|
||||
|
||||
# first we need to calculate the distance to each centroid from our data
|
||||
for k in range(n_clusters):
|
||||
for n in range(n_samples):
|
||||
dist = 0
|
||||
for d in range(dimensions):
|
||||
dist += np.abs(data[n, d] - centroids[k, d])**2
|
||||
distances[n, k] = dist
|
||||
|
||||
# we initialize an array to keep track of to which cluster each point belongs
|
||||
# the way we set it up here the index tracks which point and the value which
|
||||
# cluster the point belongs to
|
||||
cluster_labels = np.zeros(n_samples, dtype='int')
|
||||
|
||||
# next we loop through our samples and for every point assign it to the cluster
|
||||
# to which it has the smallest distance to
|
||||
for n in range(n_samples):
|
||||
# tracking variables (all of this is basically just an argmin)
|
||||
smallest = 1e10
|
||||
smallest_row_index = 1e10
|
||||
for k in range(n_clusters):
|
||||
if distances[n, k] < smallest:
|
||||
smallest = distances[n, k]
|
||||
smallest_row_index = k
|
||||
|
||||
cluster_labels[n] = smallest_row_index
|
||||
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Plotting =====
|
||||
!bc pycod
|
||||
fig = plt.figure()
|
||||
ax = fig.add_subplot()
|
||||
unique_cluster_labels = np.unique(cluster_labels)
|
||||
for i in unique_cluster_labels:
|
||||
ax.scatter(data[cluster_labels == i, 0],
|
||||
data[cluster_labels == i, 1],
|
||||
label = i,
|
||||
alpha = 0.2)
|
||||
ax.scatter(centroids[:, 0], centroids[:, 1], c='black')
|
||||
|
||||
ax.set_title("First Grouping of Points to Centroids")
|
||||
|
||||
plt.show()
|
||||
!ec
|
||||
|
||||
So what do we have so far? We have 'picked' $k$ centroids at random from our
|
||||
data points. There are other ways of more intelligently choosing their
|
||||
initializations, however for our purposes randomly is fine. Then we have
|
||||
initialized an array 'distances' which holds the information of the distance,
|
||||
*or dissimilarity*, of every point to of our centroids. Finally, we have
|
||||
initialized an array 'cluster_labels' which according to our distances array
|
||||
holds the information of to which centroid every point is assigned. This was the
|
||||
first pass of our algorithm. Essentially, all we need to do now is repeat the
|
||||
distance and assignment steps above until we have reached a desired convergence
|
||||
or a maximum amount of iterations.
|
||||
|
||||
!split
|
||||
===== Continuing =====
|
||||
|
||||
!bc pycod
|
||||
|
||||
max_iterations = 100
|
||||
tolerance = 1e-8
|
||||
|
||||
for iteration in range(max_iterations):
|
||||
prev_centroids = centroids.copy()
|
||||
for k in range(n_clusters):
|
||||
# this array will be used to update our centroid positions
|
||||
vector_mean = np.zeros(dimensions)
|
||||
mean_divisor = 0
|
||||
for n in range(n_samples):
|
||||
if cluster_labels[n] == k:
|
||||
vector_mean += data[n, :]
|
||||
mean_divisor += 1
|
||||
|
||||
# update according to the k means
|
||||
centroids[k, :] = vector_mean / mean_divisor
|
||||
|
||||
# we find the dissimilarity
|
||||
for k in range(n_clusters):
|
||||
for n in range(n_samples):
|
||||
dist = 0
|
||||
for d in range(dimensions):
|
||||
dist += np.abs(data[n, d] - centroids[k, d])**2
|
||||
distances[n, k] = dist
|
||||
|
||||
# assign each point
|
||||
for n in range(n_samples):
|
||||
smallest = 1e10
|
||||
smallest_row_index = 1e10
|
||||
for k in range(n_clusters):
|
||||
if distances[n, k] < smallest:
|
||||
smallest = distances[n, k]
|
||||
smallest_row_index = k
|
||||
|
||||
cluster_labels[n] = smallest_row_index
|
||||
|
||||
# convergence criteria
|
||||
centroid_difference = np.sum(np.abs(centroids - prev_centroids))
|
||||
if centroid_difference < tolerance:
|
||||
print(f'Converged at iteration {iteration}')
|
||||
break
|
||||
|
||||
elif iteration == max_iterations:
|
||||
print(f'Did not converge in {max_iterations} iterations')
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Wrapping it up =====
|
||||
We now have a simple , un-optimized $k$-means
|
||||
clustering implementation. Lets plot the final result
|
||||
|
||||
!bc pycod
|
||||
fig = plt.figure()
|
||||
ax = fig.add_subplot()
|
||||
unique_cluster_labels = np.unique(cluster_labels)
|
||||
for i in unique_cluster_labels:
|
||||
ax.scatter(data[cluster_labels == i, 0],
|
||||
data[cluster_labels == i, 1],
|
||||
label = i,
|
||||
alpha = 0.2)
|
||||
ax.scatter(centroids[:, 0], centroids[:, 1], c='black')
|
||||
|
||||
ax.set_title("Final Result of K-means Clustering")
|
||||
|
||||
plt.show()
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
def naive_kmeans(data, n_clusters=4, max_iterations=100, tolerance=1e-8):
|
||||
start_time = time.time()
|
||||
|
||||
n_samples, dimensions = data.shape
|
||||
n_clusters = 4
|
||||
#np.random.seed(2021)
|
||||
centroids = data[np.random.choice(n_samples, n_clusters, replace=False), :]
|
||||
distances = np.zeros((n_samples, n_clusters))
|
||||
|
||||
for k in range(n_clusters):
|
||||
for n in range(n_samples):
|
||||
dist = 0
|
||||
for d in range(dimensions):
|
||||
dist += np.abs(data[n, d] - centroids[k, d])**2
|
||||
distances[n, k] = dist
|
||||
|
||||
cluster_labels = np.zeros(n_samples, dtype='int')
|
||||
|
||||
for n in range(n_samples):
|
||||
smallest = 1e10
|
||||
smallest_row_index = 1e10
|
||||
for k in range(n_clusters):
|
||||
if distances[n, k] < smallest:
|
||||
smallest = distances[n, k]
|
||||
smallest_row_index = k
|
||||
|
||||
cluster_labels[n] = smallest_row_index
|
||||
|
||||
for iteration in range(max_iterations):
|
||||
prev_centroids = centroids.copy()
|
||||
for k in range(n_clusters):
|
||||
vector_mean = np.zeros(dimensions)
|
||||
mean_divisor = 0
|
||||
for n in range(n_samples):
|
||||
if cluster_labels[n] == k:
|
||||
vector_mean += data[n, :]
|
||||
mean_divisor += 1
|
||||
|
||||
centroids[k, :] = vector_mean / mean_divisor
|
||||
|
||||
for k in range(n_clusters):
|
||||
for n in range(n_samples):
|
||||
dist = 0
|
||||
for d in range(dimensions):
|
||||
dist += np.abs(data[n, d] - centroids[k, d])**2
|
||||
distances[n, k] = dist
|
||||
|
||||
for n in range(n_samples):
|
||||
smallest = 1e10
|
||||
smallest_row_index = 1e10
|
||||
for k in range(n_clusters):
|
||||
if distances[n, k] < smallest:
|
||||
smallest = distances[n, k]
|
||||
smallest_row_index = k
|
||||
|
||||
cluster_labels[n] = smallest_row_index
|
||||
|
||||
centroid_difference = np.sum(np.abs(centroids - prev_centroids))
|
||||
if centroid_difference < tolerance:
|
||||
print(f'Converged at iteration {iteration}')
|
||||
print(f'Runtime: {time.time() - start_time} seconds')
|
||||
|
||||
return cluster_labels, centroids
|
||||
|
||||
print(f'Did not converge in {max_iterations} iterations')
|
||||
print(f'Runtime: {time.time() - start_time} seconds')
|
||||
|
||||
return cluster_labels, centroids
|
||||
|
||||
!ec
|
||||
|
||||
Decision trees and wrapping up PCA.
|
||||
|
||||
!split
|
||||
===== Decision trees, overarching aims =====
|
||||
|
||||
Reference in New Issue
Block a user