From 46aa974be94a0e2497f313685eed8feed1f9b310 Mon Sep 17 00:00:00 2001 From: mhjensen Date: Mon, 21 Oct 2019 13:13:16 +0200 Subject: [PATCH] update and correcting typos --- doc/pub/DimRed/html/._DimRed-bs000.html | 2 +- doc/pub/DimRed/html/._DimRed-bs008.html | 4 +- doc/pub/DimRed/html/._DimRed-bs009.html | 27 +- doc/pub/DimRed/html/._DimRed-bs010.html | 277 ++++++++++++++++++ doc/pub/DimRed/html/._DimRed-bs011.html | 265 +++++++++++++++++ doc/pub/DimRed/html/._DimRed-bs012.html | 267 +++++++++++++++++ doc/pub/DimRed/html/._DimRed-bs013.html | 249 ++++++++++++++++ doc/pub/DimRed/html/._DimRed-bs014.html | 285 +++++++++++++++++++ doc/pub/DimRed/html/._DimRed-bs015.html | 227 +++++++++++++++ doc/pub/DimRed/html/._DimRed-bs016.html | 225 +++++++++++++++ doc/pub/DimRed/html/._DimRed-bs017.html | 234 +++++++++++++++ doc/pub/DimRed/html/._DimRed-bs018.html | 254 +++++++++++++++++ doc/pub/DimRed/html/._DimRed-bs019.html | 248 ++++++++++++++++ doc/pub/DimRed/html/._DimRed-bs020.html | 246 ++++++++++++++++ doc/pub/DimRed/html/._DimRed-bs021.html | 227 +++++++++++++++ doc/pub/DimRed/html/._DimRed-bs022.html | 230 +++++++++++++++ doc/pub/DimRed/html/._DimRed-bs023.html | 243 ++++++++++++++++ doc/pub/DimRed/html/._DimRed-bs024.html | 224 +++++++++++++++ doc/pub/DimRed/html/._DimRed-bs025.html | 283 ++++++++++++++++++ doc/pub/DimRed/html/DimRed-bs.html | 2 +- doc/pub/DimRed/html/DimRed-reveal.html | 49 ++-- doc/pub/DimRed/html/DimRed-solarized.html | 49 ++-- doc/pub/DimRed/html/DimRed.html | 49 ++-- doc/pub/DimRed/ipynb/DimRed.ipynb | 48 ++-- doc/pub/DimRed/ipynb/ipynb-DimRed-src.tar.gz | Bin 191 -> 191 bytes doc/pub/DimRed/pdf/DimRed-minted.pdf | Bin 236978 -> 237104 bytes doc/src/DimRed/DimRed.do.txt | 46 +-- 27 files changed, 4130 insertions(+), 130 deletions(-) create mode 100644 doc/pub/DimRed/html/._DimRed-bs010.html create mode 100644 doc/pub/DimRed/html/._DimRed-bs011.html create mode 100644 doc/pub/DimRed/html/._DimRed-bs012.html create mode 100644 doc/pub/DimRed/html/._DimRed-bs013.html create mode 100644 doc/pub/DimRed/html/._DimRed-bs014.html create mode 100644 doc/pub/DimRed/html/._DimRed-bs015.html create mode 100644 doc/pub/DimRed/html/._DimRed-bs016.html create mode 100644 doc/pub/DimRed/html/._DimRed-bs017.html create mode 100644 doc/pub/DimRed/html/._DimRed-bs018.html create mode 100644 doc/pub/DimRed/html/._DimRed-bs019.html create mode 100644 doc/pub/DimRed/html/._DimRed-bs020.html create mode 100644 doc/pub/DimRed/html/._DimRed-bs021.html create mode 100644 doc/pub/DimRed/html/._DimRed-bs022.html create mode 100644 doc/pub/DimRed/html/._DimRed-bs023.html create mode 100644 doc/pub/DimRed/html/._DimRed-bs024.html create mode 100644 doc/pub/DimRed/html/._DimRed-bs025.html diff --git a/doc/pub/DimRed/html/._DimRed-bs000.html b/doc/pub/DimRed/html/._DimRed-bs000.html index 74d5b19a6..adb8a3e0c 100644 --- a/doc/pub/DimRed/html/._DimRed-bs000.html +++ b/doc/pub/DimRed/html/._DimRed-bs000.html @@ -190,7 +190,7 @@ MathJax.Hub.Config({
[2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University

-

Oct 20, 2019

+

Oct 21, 2019


diff --git a/doc/pub/DimRed/html/._DimRed-bs008.html b/doc/pub/DimRed/html/._DimRed-bs008.html index cbc06114e..399fd0ef5 100644 --- a/doc/pub/DimRed/html/._DimRed-bs008.html +++ b/doc/pub/DimRed/html/._DimRed-bs008.html @@ -177,9 +177,9 @@ MathJax.Hub.Config({ We have a data set defined by a design/feature matrix \( \boldsymbol{X} \) (see below for its definition)

diff --git a/doc/pub/DimRed/html/._DimRed-bs009.html b/doc/pub/DimRed/html/._DimRed-bs009.html index b8821f107..a835f1361 100644 --- a/doc/pub/DimRed/html/._DimRed-bs009.html +++ b/doc/pub/DimRed/html/._DimRed-bs009.html @@ -173,29 +173,32 @@ MathJax.Hub.Config({

Introducing the Covariance and Correlation functions

+

+Before we discuss the PCA theorem, we need to remind ourselves about the definition of the covariance and the correlation function. +

Suppose we have defined two vectors \( \hat{x} \) and \( \hat{y} \) with \( n \) elements each. The covariance matrix \( \boldsymbol{C} \) is defined as $$ -\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} cov[\boldsymbol{x},\boldsymbol{x}] & cov[\boldsymbol{x},\boldsymbol{y}] \\ - cov[\boldsymbol{y},\boldsymbol{x}] & cov[\boldsymbol{y},\boldsymbol{y}] \\ +\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{cov}[\boldsymbol{x},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{cov}[\boldsymbol{y},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{y},\boldsymbol{y}] \\ \end{bmatrix}, $$ where for example $$ -cov[\boldsymbol{x},\boldsymbol{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). +\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). $$ -With this definition and recalling that the variance is +With this definition and recalling that the variance is defined as $$ -var[\boldsymbol{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2, +\mathrm{var}[\boldsymbol{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2, $$ -we can rewrite the covariance matrix in this case as +we can rewrite the covariance matrix as $$ -\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} var[\boldsymbol{x}] & cov[\boldsymbol{x},\boldsymbol{y}] \\ - cov[\boldsymbol{x},\boldsymbol{y}] & var[\boldsymbol{y}] \\ +\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{var}[\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] & \mathrm{var}[\boldsymbol{y}] \\ \end{bmatrix}, $$ @@ -207,18 +210,18 @@ introducing instead the correlation matrix defined via the so-called correlation function $$ -corr[\boldsymbol{x},\boldsymbol{y}]=\frac{cov[\boldsymbol{x},\boldsymbol{y}]}{\sqrt{var[\boldsymbol{x}]\var[\boldsymbol{y}]}}. +\mathrm{corr}[\boldsymbol{x},\boldsymbol{y}]=\frac{\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}]}{\sqrt{\mathrm{var}[\boldsymbol{x}] \mathrm{var}[\boldsymbol{y}]}}. $$

-The correlation function is then given by values \( corr[\boldsymbol{x},\boldsymbol{y}] +The correlation function is then given by values \( \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] \in [-1,1] \). This avoids eventual problems with too large values. We can then define the correlation matrix for the two vectors \( \boldsymbol{x} \) and \( \boldsymbol{y} \) as $$ -\boldsymbol{K}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} 1 & corr[\boldsymbol{x},\boldsymbol{y}] \\ - corr[\boldsymbol{y},\boldsymbol{x}] & 1 \\ +\boldsymbol{K}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} 1 & \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{corr}[\boldsymbol{y},\boldsymbol{x}] & 1 \\ \end{bmatrix}, $$ diff --git a/doc/pub/DimRed/html/._DimRed-bs010.html b/doc/pub/DimRed/html/._DimRed-bs010.html new file mode 100644 index 000000000..ece2f0cc0 --- /dev/null +++ b/doc/pub/DimRed/html/._DimRed-bs010.html @@ -0,0 +1,277 @@ + + + + + + + + +Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction + + + + + + + + + + + + + + + + + + + + + + + +

+
+ + Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Correlation Function and Design/Feature Matrix

+ +

+In our derivation of the various regression algorithms like Ordinary Least Squares or Ridge regression we defined the design/feature matrix \( \boldsymbol{X} \) as +$$ +\boldsymbol{X}=\begin{bmatrix} +x_{0,0} & x_{0,1} & x_{0,2}& \dots & \dots x_{0,p-1}\\ +x_{1,0} & x_{1,1} & x_{1,2}& \dots & \dots x_{1,p-1}\\ +x_{2,0} & x_{2,1} & x_{2,2}& \dots & \dots x_{2,p-1}\\ +\dots & \dots & \dots & \dots \dots & \dots \\ +x_{n-2,0} & x_{n-2,1} & x_{n-2,2}& \dots & \dots x_{n-2,p-1}\\ +x_{n-1,0} & x_{n-1,1} & x_{n-1,2}& \dots & \dots x_{n-1,p-1}\\ +\end{bmatrix}, +$$ + +with \( \boldsymbol{X}\in {\mathbb{R}}^{n\times p} \), with the predictors/features \( p \) refering to the column numbers and the +entries \( n \) being the row elements. +We can rewrite the design/feature matrix in terms of its column vectors as +$$ +\boldsymbol{X}=\begin{bmatrix} \boldsymbol{x}_0 & \boldsymbol{x}_0 & \boldsymbol{x}_0 & \dots & \dots & \boldsymbol{x}_{p-1}\end{bmatrix}, +$$ + +with a given vector +$$ +\boldsymbol{x}_i^T = \begin{bmatrix}x_{0,i} & x_{1,i} & x_{2,i}& \dots & \dots x_{n-1,i}\end{bmatrix}. +$$ + +

+With these definitions, we can now rewrite our \( 2\times 2 \) correaltion/covariance matrix in terms of a moe general design/feature matrix \( \boldsymbol{X}\in {\mathbb{R}}^{n\times p} \). This leads to a \( p\times p \) covariance matrix for the vectors \( \boldsymbol{x}_i \) with \( i =0,1,\dots,p-1 \) +$$ +\boldsymbol{C}[\boldsymbol{x}] = \begin{bmatrix} +\mathrm{var}[\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ +\mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_0] & \mathrm{var}[\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ +\mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_1] & \mathrm{var}[\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & \mathrm{var}[\boldsymbol{x}_{p-1}]\\ +\end{bmatrix}, +$$ + +and the correlation matrix +$$ +\boldsymbol{K}[\boldsymbol{x}] = \begin{bmatrix} +1 & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ +\mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_0] & 1 & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ +\mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_1] & 1 & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & 1\\ +\end{bmatrix}, +$$ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DimRed/html/._DimRed-bs011.html b/doc/pub/DimRed/html/._DimRed-bs011.html new file mode 100644 index 000000000..efb1df331 --- /dev/null +++ b/doc/pub/DimRed/html/._DimRed-bs011.html @@ -0,0 +1,265 @@ + + + + + + + + +Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Covariance Matrix Examples

+ +

+The Numpy function np.cov calculates the covariance elements using +the factor \( 1/(n-1) \) instead of \( 1/n \) since it assumes we do not have +the exact mean values. The following simple function uses the +np.vstack function which takes each vector of dimension \( 1\times n \) +and produces a \( 2\times n \) matrix \( \boldsymbol{W} \) + +$$ +\boldsymbol{W} = \begin{bmatrix} x_0 & y_0 \\ + x_1 & y_1 \\ + x_2 & y_2\\ + \dots & \dots \\ + x_{n-2} & y_{n-2}\\ + x_{n-1} & y_{n-1} & + \end{bmatrix}, +$$ + +

+which in turn is converted into into the \( 2\times 2 \) covariance matrix +\( \boldsymbol{C} \) via the Numpy function np.cov(). We note that we can also calculate +the mean value of each set of samples \( \boldsymbol{x} \) etc using the Numpy +function np.mean(x). We can also extract the eigenvalues of the +covariance matrix through the np.linalg.eig() function. + +

+ + +

# Importing various packages
+import numpy as np
+n = 100
+x = np.random.normal(size=n)
+print(np.mean(x))
+y = 4+3*x+np.random.normal(size=n)
+print(np.mean(y))
+W = np.vstack((x, y))
+C = np.cov(W)
+print(C)
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DimRed/html/._DimRed-bs012.html b/doc/pub/DimRed/html/._DimRed-bs012.html new file mode 100644 index 000000000..6db7b8fba --- /dev/null +++ b/doc/pub/DimRed/html/._DimRed-bs012.html @@ -0,0 +1,267 @@ + + + + + + + + +Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Correlation Matrix

+ +

+The previous example can be converted into the correlation matrix by +simply scaling the matrix elements with the variances. We should also +subtract the mean values for each column. This leads to the following +code which sets up the correlations matrix for the previous example in +a more brute force way. Here we scale the mean values for each column of the design matrix, calculate the relevant mean values and variances and then finally set up the \( 2\times 2 \) correlation matrix (since we have only two vectors). + +

+ + +

import numpy as np
+n = 100
+# define two vectors                                                                                           
+x = np.random.random(size=n)
+y = 4+3*x+np.random.normal(size=n)
+#scaling the x and y vectors                                                                                   
+x = x - np.mean(x)
+y = y - np.mean(y)
+variance_x = np.sum(x@x)/n
+variance_y = np.sum(y@y)/n
+print(variance_x)
+print(variance_y)
+cov_xy = np.sum(x@y)/n
+cov_xx = np.sum(x@x)/n
+cov_yy = np.sum(y@y)/n
+C = np.zeros((2,2))
+C[0,0]= cov_xx/variance_x
+C[1,1]= cov_yy/variance_y
+C[0,1]= cov_xy/np.sqrt(variance_y*variance_x)
+C[1,0]= C[0,1]
+print(C)
+
+

+We see that the matrix elements along the diagonal are one as they +should be and that the matrix is symmetric. Furthermore, diagonalizing +this matrix we easily see that it is a positive definite matrix. + +

+The above procedure with numpy can be made more compact if we use pandas. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DimRed/html/._DimRed-bs013.html b/doc/pub/DimRed/html/._DimRed-bs013.html new file mode 100644 index 000000000..3a6c4aece --- /dev/null +++ b/doc/pub/DimRed/html/._DimRed-bs013.html @@ -0,0 +1,249 @@ + + + + + + + + +Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Correlation Matrix with Pandas

+ +

+We whow here how we can set up the correlation matrix using pandas, as done in this simple code +

+ + +

import numpy as np
+import pandas as pd
+n = 10
+x = np.random.normal(size=n)
+x = x - np.mean(x)
+y = 4+3*x+np.random.normal(size=n)
+y = y - np.mean(y)
+X = (np.vstack((x, y))).T
+print(X)
+Xpd = pd.DataFrame(X)
+print(Xpd)
+correlation_matrix = Xpd.corr()
+print(correlation_matrix)
+
+

+We expand this model to the Franke function discussed above. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DimRed/html/._DimRed-bs014.html b/doc/pub/DimRed/html/._DimRed-bs014.html new file mode 100644 index 000000000..2ed83ed7c --- /dev/null +++ b/doc/pub/DimRed/html/._DimRed-bs014.html @@ -0,0 +1,285 @@ + + + + + + + + +Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Correlation Matrix with Pandas and the Franke function

+ +

+ + +

# Common imports
+import numpy as np
+import pandas as pd
+
+
+def FrankeFunction(x,y):
+	term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
+	term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
+	term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
+	term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
+	return term1 + term2 + term3 + term4
+
+
+def create_X(x, y, n ):
+	if len(x.shape) > 1:
+		x = np.ravel(x)
+		y = np.ravel(y)
+
+	N = len(x)
+	l = int((n+1)*(n+2)/2)		# Number of elements in beta
+	X = np.ones((N,l))
+
+	for i in range(1,n+1):
+		q = int((i)*(i+1)/2)
+		for k in range(i+1):
+			X[:,q+k] = (x**(i-k))*(y**k)
+
+	return X
+
+
+# Making meshgrid of datapoints and compute Franke's function
+n = 4
+N = 100
+x = np.sort(np.random.uniform(0, 1, N))
+y = np.sort(np.random.uniform(0, 1, N))
+z = FrankeFunction(x, y)
+X = create_X(x, y, n=n)    
+
+Xpd = pd.DataFrame(X)
+# subtract the mean values and set up the covariance matrix
+Xpd = Xpd - Xpd.mean()
+covariance_matrix = Xpd.cov()
+print(covariance_matrix)
+
+

+We note here that the covariance is zero for the first rows and +columns since all matrix elements in the design matrix were set to one +(we are fitting the function in terms of a polynomial of degree \( n \)). + +

+This means that the variance for these elements will be zero and will +cause problems when we set up the correlation matrix. We can simply +drop these elements as follows and then construct the correlation +matrix. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DimRed/html/._DimRed-bs015.html b/doc/pub/DimRed/html/._DimRed-bs015.html new file mode 100644 index 000000000..e338e1353 --- /dev/null +++ b/doc/pub/DimRed/html/._DimRed-bs015.html @@ -0,0 +1,227 @@ + + + + + + + + +Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Classical PCA Theorem

+ +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DimRed/html/._DimRed-bs016.html b/doc/pub/DimRed/html/._DimRed-bs016.html new file mode 100644 index 000000000..bf52a93b2 --- /dev/null +++ b/doc/pub/DimRed/html/._DimRed-bs016.html @@ -0,0 +1,225 @@ + + + + + + + + +Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Prof of the PCA Theorem

+ +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DimRed/html/._DimRed-bs017.html b/doc/pub/DimRed/html/._DimRed-bs017.html new file mode 100644 index 000000000..10535b5c8 --- /dev/null +++ b/doc/pub/DimRed/html/._DimRed-bs017.html @@ -0,0 +1,234 @@ + + + + + + + + +Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Getting started with PCA

+ +

+ + +

# Now add PCA
+from sklearn.decomposition import PCA
+pca = PCA(n_components = 2)
+pca.fit(X_train_scaled)
+
+X_pca = pca.transform(X_train_scaled)
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DimRed/html/._DimRed-bs018.html b/doc/pub/DimRed/html/._DimRed-bs018.html new file mode 100644 index 000000000..bcec440d0 --- /dev/null +++ b/doc/pub/DimRed/html/._DimRed-bs018.html @@ -0,0 +1,254 @@ + + + + + + + + +Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Principal Component Analysis

+
+
+

+Principal Component Analysis (PCA) is by far the most popular dimensionality reduction algorithm. +First it identifies the hyperplane that lies closest to the data, and then it projects the data onto it. + +

+The following Python code uses NumPy’s svd() function to obtain all the principal components of the +training set, then extracts the first two principal components +

+ + +

X_centered = X - X.mean(axis=0)
+U, s, V = np.linalg.svd(X_centered)
+c1 = V.T[:, 0]
+c2 = V.T[:, 1]
+
+

+PCA assumes that the dataset is centered around the origin. Scikit-Learn’s PCA classes take care of centering +the data for you. However, if you implement PCA yourself (as in the preceding example), or if you use other libraries, don’t +forget to center the data first. + +

+Once you have identified all the principal components, you can reduce the dimensionality of the dataset +down to \( d \) dimensions by projecting it onto the hyperplane defined by the first \( d \) principal components. +Selecting this hyperplane ensures that the projection will preserve as much variance as possible. +

+ + +

W2 = V.T[:, :2]
+X2D = X_centered.dot(W2)
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DimRed/html/._DimRed-bs019.html b/doc/pub/DimRed/html/._DimRed-bs019.html new file mode 100644 index 000000000..1e3679a62 --- /dev/null +++ b/doc/pub/DimRed/html/._DimRed-bs019.html @@ -0,0 +1,248 @@ + + + + + + + + +Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction +
+ +
+ +
+
+
+ +
+ +

 

 

 

+ + + + +

PCA and scikit-learn

+ +

+Scikit-Learn’s PCA class implements PCA using SVD decomposition just like we did before. The +following code applies PCA to reduce the dimensionality of the dataset down to two dimensions (note +that it automatically takes care of centering the data): +

+ + +

from sklearn.decomposition import PCA
+pca = PCA(n_components = 2)
+X2D = pca.fit_transform(X)
+
+

+After fitting the PCA transformer to the dataset, you can access the principal components using the +components variable (note that it contains the PCs as horizontal vectors, so, for example, the first +principal component is equal to +

+ + +

pca.components_.T[:, 0]).
+
+

+Another very useful piece of information is the explained variance ratio of each principal component, +available via the \( explained\_variance\_ratio \) variable. It indicates the proportion of the dataset’s +variance that lies along the axis of each principal component. +More material to come here. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DimRed/html/._DimRed-bs020.html b/doc/pub/DimRed/html/._DimRed-bs020.html new file mode 100644 index 000000000..758b5efa4 --- /dev/null +++ b/doc/pub/DimRed/html/._DimRed-bs020.html @@ -0,0 +1,246 @@ + + + + + + + + +Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction +
+ +
+ +
+
+
+ +
+ +

 

 

 

+ + + + +

More on the PCA

+ +

+Instead of arbitrarily choosing the number of dimensions to reduce down to, it is generally preferable to +choose the number of dimensions that add up to a sufficiently large portion of the variance (e.g., 95%). +Unless, of course, you are reducing dimensionality for data visualization — in that case you will +generally want to reduce the dimensionality down to 2 or 3. +The following code computes PCA without reducing dimensionality, then computes the minimum number +of dimensions required to preserve 95% of the training set’s variance: +

+ + +

pca = PCA()
+pca.fit(X)
+cumsum = np.cumsum(pca.explained_variance_ratio_)
+d = np.argmax(cumsum >= 0.95) + 1
+
+

+You could then set \( n\_components=d \) and run PCA again. However, there is a much better option: instead +of specifying the number of principal components you want to preserve, you can set \( n\_components \) to be +a float between 0.0 and 1.0, indicating the ratio of variance you wish to preserve: +

+ + +

pca = PCA(n_components=0.95)
+X_reduced = pca.fit_transform(X)
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DimRed/html/._DimRed-bs021.html b/doc/pub/DimRed/html/._DimRed-bs021.html new file mode 100644 index 000000000..05480d8f8 --- /dev/null +++ b/doc/pub/DimRed/html/._DimRed-bs021.html @@ -0,0 +1,227 @@ + + + + + + + + +Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Incremental PCA

+ +

+One problem with the preceding implementation of PCA is that it requires the whole training set to fit in +memory in order for the SVD algorithm to run. Fortunately, Incremental PCA (IPCA) algorithms have +been developed: you can split the training set into mini-batches and feed an IPCA algorithm one minibatch +at a time. This is useful for large training sets, and also to apply PCA online (i.e., on the fly, as new +instances arrive). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DimRed/html/._DimRed-bs022.html b/doc/pub/DimRed/html/._DimRed-bs022.html new file mode 100644 index 000000000..4490cfad1 --- /dev/null +++ b/doc/pub/DimRed/html/._DimRed-bs022.html @@ -0,0 +1,230 @@ + + + + + + + + +Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Randomized PCA

+ +

+Scikit-Learn offers yet another option to perform PCA, called Randomized PCA. This is a stochastic +algorithm that quickly finds an approximation of the first d principal components. Its computational +complexity is \( O(m \times d^2)+O(d^3) \), instead of \( O(m \times n^2) + O(n^3) \), so it is dramatically faster than the +previous algorithms when \( d \) is much smaller than \( n \). + +

+

+ + + +

+

+ +

+ + + + + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DimRed/html/._DimRed-bs023.html b/doc/pub/DimRed/html/._DimRed-bs023.html new file mode 100644 index 000000000..9aee50d3b --- /dev/null +++ b/doc/pub/DimRed/html/._DimRed-bs023.html @@ -0,0 +1,243 @@ + + + + + + + + +Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Kernel PCA

+
+
+

+ +

+The kernel trick is a mathematical technique that implicitly maps instances into a +very high-dimensional space (called the feature space), enabling nonlinear classification and regression +with Support Vector Machines. Recall that a linear decision boundary in the high-dimensional feature +space corresponds to a complex nonlinear decision boundary in the original space. +It turns out that the same trick can be applied to PCA, making it possible to perform complex nonlinear +projections for dimensionality reduction. This is called Kernel PCA (kPCA). It is often good at +preserving clusters of instances after projection, or sometimes even unrolling datasets that lie close to a +twisted manifold. +For example, the following code uses Scikit-Learn’s KernelPCA class to perform kPCA with an +

+ + +

from sklearn.decomposition import KernelPCA
+rbf_pca = KernelPCA(n_components = 2, kernel="rbf", gamma=0.04)
+X_reduced = rbf_pca.fit_transform(X)
+
+

+

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DimRed/html/._DimRed-bs024.html b/doc/pub/DimRed/html/._DimRed-bs024.html new file mode 100644 index 000000000..ace8ed826 --- /dev/null +++ b/doc/pub/DimRed/html/._DimRed-bs024.html @@ -0,0 +1,224 @@ + + + + + + + + +Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

LLE

+ +

+Locally Linear Embedding (LLE) is another very powerful nonlinear dimensionality reduction +(NLDR) technique. It is a Manifold Learning technique that does not rely on projections like the previous +algorithms. In a nutshell, LLE works by first measuring how each training instance linearly relates to its +closest neighbors (c.n.), and then looking for a low-dimensional representation of the training set where +these local relationships are best preserved (more details shortly). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DimRed/html/._DimRed-bs025.html b/doc/pub/DimRed/html/._DimRed-bs025.html new file mode 100644 index 000000000..a053bea68 --- /dev/null +++ b/doc/pub/DimRed/html/._DimRed-bs025.html @@ -0,0 +1,283 @@ + + + + + + + + +Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Other techniques

+ +

+There are many other dimensionality reduction techniques, several of which are available in Scikit-Learn. + +

+Here are some of the most popular: + +

+ +Here are other examples where we use the DataFrame functionality to handle arrays, now with more interesting features for us, namely numbers. We set up a matrix +of dimensionality \( 10\times 5 \) and compute the mean value and standard deviation of each column. Similarly, we can perform mathematial operations like squaring the matrix elements and many other operations. +

+ + +

import numpy as np
+import pandas as pd
+from IPython.display import display
+np.random.seed(100)
+# setting up a 10 x 5 matrix
+rows = 10
+cols = 5
+a = np.random.randn(rows,cols)
+df = pd.DataFrame(a)
+display(df)
+print(df.mean())
+print(df.std())
+display(df**2)
+
+

+Thereafter we can select specific columns only and plot final results +

+ + +

df.columns = ['First', 'Second', 'Third', 'Fourth', 'Fifth']
+df.index = np.arange(10)
+
+display(df)
+print(df['Second'].mean() )
+print(df.info())
+print(df.describe())
+
+from pylab import plt, mpl
+plt.style.use('seaborn')
+mpl.rcParams['font.family'] = 'serif'
+
+df.cumsum().plot(lw=2.0, figsize=(10,6))
+plt.show()
+
+
+df.plot.bar(figsize=(10,6), rot=15)
+plt.show()
+
+

+We can produce a \( 4\times 4 \) matrix +

+ + +

b = np.arange(16).reshape((4,4))
+print(b)
+df1 = pd.DataFrame(b)
+print(df1)
+
+

+and many other operations. + +

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/DimRed/html/DimRed-bs.html b/doc/pub/DimRed/html/DimRed-bs.html index 74d5b19a6..adb8a3e0c 100644 --- a/doc/pub/DimRed/html/DimRed-bs.html +++ b/doc/pub/DimRed/html/DimRed-bs.html @@ -190,7 +190,7 @@ MathJax.Hub.Config({
[2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University

-

Oct 20, 2019

+

Oct 21, 2019


diff --git a/doc/pub/DimRed/html/DimRed-reveal.html b/doc/pub/DimRed/html/DimRed-reveal.html index 0886daa11..845860690 100644 --- a/doc/pub/DimRed/html/DimRed-reveal.html +++ b/doc/pub/DimRed/html/DimRed-reveal.html @@ -148,7 +148,7 @@ MathJax.Hub.Config({

[2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University

 
-

Oct 20, 2019

+

Oct 21, 2019


@@ -514,9 +514,9 @@ applications. We have a data set defined by a design/feature matrix \( \boldsymbol{X} \) (see below for its definition)

@@ -524,13 +524,16 @@ We have a data set defined by a design/feature matrix \( \boldsymbol{X} \) (see

Introducing the Covariance and Correlation functions

+

+Before we discuss the PCA theorem, we need to remind ourselves about the definition of the covariance and the correlation function. +

Suppose we have defined two vectors \( \hat{x} \) and \( \hat{y} \) with \( n \) elements each. The covariance matrix \( \boldsymbol{C} \) is defined as

 
$$ -\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} cov[\boldsymbol{x},\boldsymbol{x}] & cov[\boldsymbol{x},\boldsymbol{y}] \\ - cov[\boldsymbol{y},\boldsymbol{x}] & cov[\boldsymbol{y},\boldsymbol{y}] \\ +\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{cov}[\boldsymbol{x},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{cov}[\boldsymbol{y},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{y},\boldsymbol{y}] \\ \end{bmatrix}, $$

 
@@ -538,22 +541,22 @@ $$ where for example

 
$$ -cov[\boldsymbol{x},\boldsymbol{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). +\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). $$

 
-With this definition and recalling that the variance is +With this definition and recalling that the variance is defined as

 
$$ -var[\boldsymbol{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2, +\mathrm{var}[\boldsymbol{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2, $$

 
-we can rewrite the covariance matrix in this case as +we can rewrite the covariance matrix as

 
$$ -\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} var[\boldsymbol{x}] & cov[\boldsymbol{x},\boldsymbol{y}] \\ - cov[\boldsymbol{x},\boldsymbol{y}] & var[\boldsymbol{y}] \\ +\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{var}[\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] & \mathrm{var}[\boldsymbol{y}] \\ \end{bmatrix}, $$

 
@@ -567,20 +570,20 @@ correlation function

 
$$ -corr[\boldsymbol{x},\boldsymbol{y}]=\frac{cov[\boldsymbol{x},\boldsymbol{y}]}{\sqrt{var[\boldsymbol{x}]\var[\boldsymbol{y}]}}. +\mathrm{corr}[\boldsymbol{x},\boldsymbol{y}]=\frac{\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}]}{\sqrt{\mathrm{var}[\boldsymbol{x}] \mathrm{var}[\boldsymbol{y}]}}. $$

 

-The correlation function is then given by values \( corr[\boldsymbol{x},\boldsymbol{y}] +The correlation function is then given by values \( \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] \in [-1,1] \). This avoids eventual problems with too large values. We can then define the correlation matrix for the two vectors \( \boldsymbol{x} \) and \( \boldsymbol{y} \) as

 
$$ -\boldsymbol{K}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} 1 & corr[\boldsymbol{x},\boldsymbol{y}] \\ - corr[\boldsymbol{y},\boldsymbol{x}] & 1 \\ +\boldsymbol{K}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} 1 & \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{corr}[\boldsymbol{y},\boldsymbol{x}] & 1 \\ \end{bmatrix}, $$

 
@@ -629,12 +632,12 @@ With these definitions, we can now rewrite our \( 2\times 2 \) correaltion/covar

 
$$ \boldsymbol{C}[\boldsymbol{x}] = \begin{bmatrix} -var[\boldsymbol{x}_0] & cov[\boldsymbol{x}_0,\boldsymbol{x}_1] & cov[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & cov[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ -cov[\boldsymbol{x}_1,\boldsymbol{x}_0] & var[\boldsymbol{x}_1] & cov[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & cov[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ -cov[\boldsymbol{x}_2,\boldsymbol{x}_0] & cov[\boldsymbol{x}_2,\boldsymbol{x}_1] & var[\boldsymbol{x}_2] & \dots & \dots & cov[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ +\mathrm{var}[\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ +\mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_0] & \mathrm{var}[\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ +\mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_1] & \mathrm{var}[\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ \dots & \dots & \dots & \dots & \dots & \dots \\ \dots & \dots & \dots & \dots & \dots & \dots \\ -cov[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & cov[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & cov[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & var[\boldsymbol{x}_{p-1}]\\ +\mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & \mathrm{var}[\boldsymbol{x}_{p-1}]\\ \end{bmatrix}, $$

 
@@ -643,12 +646,12 @@ and the correlation matrix

 
$$ \boldsymbol{K}[\boldsymbol{x}] = \begin{bmatrix} -1 & corr[\boldsymbol{x}_0,\boldsymbol{x}_1] & corr[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & corr[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ -corr[\boldsymbol{x}_1,\boldsymbol{x}_0] & 1 & corr[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & corr[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ -corr[\boldsymbol{x}_2,\boldsymbol{x}_0] & corr[\boldsymbol{x}_2,\boldsymbol{x}_1] & 1 & \dots & \dots & corr[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ +1 & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ +\mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_0] & 1 & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ +\mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_1] & 1 & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ \dots & \dots & \dots & \dots & \dots & \dots \\ \dots & \dots & \dots & \dots & \dots & \dots \\ -corr[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & corr[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & corr[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & 1\\ +\mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & 1\\ \end{bmatrix}, $$

 
diff --git a/doc/pub/DimRed/html/DimRed-solarized.html b/doc/pub/DimRed/html/DimRed-solarized.html index c215b4a80..1f613b93b 100644 --- a/doc/pub/DimRed/html/DimRed-solarized.html +++ b/doc/pub/DimRed/html/DimRed-solarized.html @@ -155,7 +155,7 @@ MathJax.Hub.Config({

[2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University

-

Oct 20, 2019

+

Oct 21, 2019












@@ -518,38 +518,41 @@ applications. We have a data set defined by a design/feature matrix \( \boldsymbol{X} \) (see below for its definition)











Introducing the Covariance and Correlation functions

+

+Before we discuss the PCA theorem, we need to remind ourselves about the definition of the covariance and the correlation function. +

Suppose we have defined two vectors \( \hat{x} \) and \( \hat{y} \) with \( n \) elements each. The covariance matrix \( \boldsymbol{C} \) is defined as $$ -\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} cov[\boldsymbol{x},\boldsymbol{x}] & cov[\boldsymbol{x},\boldsymbol{y}] \\ - cov[\boldsymbol{y},\boldsymbol{x}] & cov[\boldsymbol{y},\boldsymbol{y}] \\ +\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{cov}[\boldsymbol{x},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{cov}[\boldsymbol{y},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{y},\boldsymbol{y}] \\ \end{bmatrix}, $$ where for example $$ -cov[\boldsymbol{x},\boldsymbol{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). +\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). $$ -With this definition and recalling that the variance is +With this definition and recalling that the variance is defined as $$ -var[\boldsymbol{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2, +\mathrm{var}[\boldsymbol{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2, $$ -we can rewrite the covariance matrix in this case as +we can rewrite the covariance matrix as $$ -\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} var[\boldsymbol{x}] & cov[\boldsymbol{x},\boldsymbol{y}] \\ - cov[\boldsymbol{x},\boldsymbol{y}] & var[\boldsymbol{y}] \\ +\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{var}[\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] & \mathrm{var}[\boldsymbol{y}] \\ \end{bmatrix}, $$ @@ -561,18 +564,18 @@ introducing instead the correlation matrix defined via the so-called correlation function $$ -corr[\boldsymbol{x},\boldsymbol{y}]=\frac{cov[\boldsymbol{x},\boldsymbol{y}]}{\sqrt{var[\boldsymbol{x}]\var[\boldsymbol{y}]}}. +\mathrm{corr}[\boldsymbol{x},\boldsymbol{y}]=\frac{\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}]}{\sqrt{\mathrm{var}[\boldsymbol{x}] \mathrm{var}[\boldsymbol{y}]}}. $$

-The correlation function is then given by values \( corr[\boldsymbol{x},\boldsymbol{y}] +The correlation function is then given by values \( \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] \in [-1,1] \). This avoids eventual problems with too large values. We can then define the correlation matrix for the two vectors \( \boldsymbol{x} \) and \( \boldsymbol{y} \) as $$ -\boldsymbol{K}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} 1 & corr[\boldsymbol{x},\boldsymbol{y}] \\ - corr[\boldsymbol{y},\boldsymbol{x}] & 1 \\ +\boldsymbol{K}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} 1 & \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{corr}[\boldsymbol{y},\boldsymbol{x}] & 1 \\ \end{bmatrix}, $$ @@ -613,24 +616,24 @@ $$ With these definitions, we can now rewrite our \( 2\times 2 \) correaltion/covariance matrix in terms of a moe general design/feature matrix \( \boldsymbol{X}\in {\mathbb{R}}^{n\times p} \). This leads to a \( p\times p \) covariance matrix for the vectors \( \boldsymbol{x}_i \) with \( i =0,1,\dots,p-1 \) $$ \boldsymbol{C}[\boldsymbol{x}] = \begin{bmatrix} -var[\boldsymbol{x}_0] & cov[\boldsymbol{x}_0,\boldsymbol{x}_1] & cov[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & cov[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ -cov[\boldsymbol{x}_1,\boldsymbol{x}_0] & var[\boldsymbol{x}_1] & cov[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & cov[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ -cov[\boldsymbol{x}_2,\boldsymbol{x}_0] & cov[\boldsymbol{x}_2,\boldsymbol{x}_1] & var[\boldsymbol{x}_2] & \dots & \dots & cov[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ +\mathrm{var}[\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ +\mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_0] & \mathrm{var}[\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ +\mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_1] & \mathrm{var}[\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ \dots & \dots & \dots & \dots & \dots & \dots \\ \dots & \dots & \dots & \dots & \dots & \dots \\ -cov[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & cov[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & cov[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & var[\boldsymbol{x}_{p-1}]\\ +\mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & \mathrm{var}[\boldsymbol{x}_{p-1}]\\ \end{bmatrix}, $$ and the correlation matrix $$ \boldsymbol{K}[\boldsymbol{x}] = \begin{bmatrix} -1 & corr[\boldsymbol{x}_0,\boldsymbol{x}_1] & corr[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & corr[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ -corr[\boldsymbol{x}_1,\boldsymbol{x}_0] & 1 & corr[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & corr[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ -corr[\boldsymbol{x}_2,\boldsymbol{x}_0] & corr[\boldsymbol{x}_2,\boldsymbol{x}_1] & 1 & \dots & \dots & corr[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ +1 & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ +\mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_0] & 1 & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ +\mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_1] & 1 & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ \dots & \dots & \dots & \dots & \dots & \dots \\ \dots & \dots & \dots & \dots & \dots & \dots \\ -corr[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & corr[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & corr[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & 1\\ +\mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & 1\\ \end{bmatrix}, $$ diff --git a/doc/pub/DimRed/html/DimRed.html b/doc/pub/DimRed/html/DimRed.html index d3244342a..c060dfc24 100644 --- a/doc/pub/DimRed/html/DimRed.html +++ b/doc/pub/DimRed/html/DimRed.html @@ -160,7 +160,7 @@ MathJax.Hub.Config({

[2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University

-

Oct 20, 2019

+

Oct 21, 2019












@@ -523,38 +523,41 @@ applications. We have a data set defined by a design/feature matrix \( \boldsymbol{X} \) (see below for its definition)











Introducing the Covariance and Correlation functions

+

+Before we discuss the PCA theorem, we need to remind ourselves about the definition of the covariance and the correlation function. +

Suppose we have defined two vectors \( \hat{x} \) and \( \hat{y} \) with \( n \) elements each. The covariance matrix \( \boldsymbol{C} \) is defined as $$ -\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} cov[\boldsymbol{x},\boldsymbol{x}] & cov[\boldsymbol{x},\boldsymbol{y}] \\ - cov[\boldsymbol{y},\boldsymbol{x}] & cov[\boldsymbol{y},\boldsymbol{y}] \\ +\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{cov}[\boldsymbol{x},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{cov}[\boldsymbol{y},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{y},\boldsymbol{y}] \\ \end{bmatrix}, $$ where for example $$ -cov[\boldsymbol{x},\boldsymbol{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). +\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). $$ -With this definition and recalling that the variance is +With this definition and recalling that the variance is defined as $$ -var[\boldsymbol{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2, +\mathrm{var}[\boldsymbol{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2, $$ -we can rewrite the covariance matrix in this case as +we can rewrite the covariance matrix as $$ -\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} var[\boldsymbol{x}] & cov[\boldsymbol{x},\boldsymbol{y}] \\ - cov[\boldsymbol{x},\boldsymbol{y}] & var[\boldsymbol{y}] \\ +\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{var}[\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] & \mathrm{var}[\boldsymbol{y}] \\ \end{bmatrix}, $$ @@ -566,18 +569,18 @@ introducing instead the correlation matrix defined via the so-called correlation function $$ -corr[\boldsymbol{x},\boldsymbol{y}]=\frac{cov[\boldsymbol{x},\boldsymbol{y}]}{\sqrt{var[\boldsymbol{x}]\var[\boldsymbol{y}]}}. +\mathrm{corr}[\boldsymbol{x},\boldsymbol{y}]=\frac{\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}]}{\sqrt{\mathrm{var}[\boldsymbol{x}] \mathrm{var}[\boldsymbol{y}]}}. $$

-The correlation function is then given by values \( corr[\boldsymbol{x},\boldsymbol{y}] +The correlation function is then given by values \( \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] \in [-1,1] \). This avoids eventual problems with too large values. We can then define the correlation matrix for the two vectors \( \boldsymbol{x} \) and \( \boldsymbol{y} \) as $$ -\boldsymbol{K}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} 1 & corr[\boldsymbol{x},\boldsymbol{y}] \\ - corr[\boldsymbol{y},\boldsymbol{x}] & 1 \\ +\boldsymbol{K}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} 1 & \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{corr}[\boldsymbol{y},\boldsymbol{x}] & 1 \\ \end{bmatrix}, $$ @@ -618,24 +621,24 @@ $$ With these definitions, we can now rewrite our \( 2\times 2 \) correaltion/covariance matrix in terms of a moe general design/feature matrix \( \boldsymbol{X}\in {\mathbb{R}}^{n\times p} \). This leads to a \( p\times p \) covariance matrix for the vectors \( \boldsymbol{x}_i \) with \( i =0,1,\dots,p-1 \) $$ \boldsymbol{C}[\boldsymbol{x}] = \begin{bmatrix} -var[\boldsymbol{x}_0] & cov[\boldsymbol{x}_0,\boldsymbol{x}_1] & cov[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & cov[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ -cov[\boldsymbol{x}_1,\boldsymbol{x}_0] & var[\boldsymbol{x}_1] & cov[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & cov[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ -cov[\boldsymbol{x}_2,\boldsymbol{x}_0] & cov[\boldsymbol{x}_2,\boldsymbol{x}_1] & var[\boldsymbol{x}_2] & \dots & \dots & cov[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ +\mathrm{var}[\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ +\mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_0] & \mathrm{var}[\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ +\mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_1] & \mathrm{var}[\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ \dots & \dots & \dots & \dots & \dots & \dots \\ \dots & \dots & \dots & \dots & \dots & \dots \\ -cov[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & cov[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & cov[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & var[\boldsymbol{x}_{p-1}]\\ +\mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & \mathrm{var}[\boldsymbol{x}_{p-1}]\\ \end{bmatrix}, $$ and the correlation matrix $$ \boldsymbol{K}[\boldsymbol{x}] = \begin{bmatrix} -1 & corr[\boldsymbol{x}_0,\boldsymbol{x}_1] & corr[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & corr[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ -corr[\boldsymbol{x}_1,\boldsymbol{x}_0] & 1 & corr[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & corr[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ -corr[\boldsymbol{x}_2,\boldsymbol{x}_0] & corr[\boldsymbol{x}_2,\boldsymbol{x}_1] & 1 & \dots & \dots & corr[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ +1 & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ +\mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_0] & 1 & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ +\mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_1] & 1 & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ \dots & \dots & \dots & \dots & \dots & \dots \\ \dots & \dots & \dots & \dots & \dots & \dots \\ -corr[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & corr[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & corr[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & 1\\ +\mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & 1\\ \end{bmatrix}, $$ diff --git a/doc/pub/DimRed/ipynb/DimRed.ipynb b/doc/pub/DimRed/ipynb/DimRed.ipynb index 0e28b6e62..f65c66146 100644 --- a/doc/pub/DimRed/ipynb/DimRed.ipynb +++ b/doc/pub/DimRed/ipynb/DimRed.ipynb @@ -10,7 +10,7 @@ " \n", "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n", "\n", - "Date: **Oct 20, 2019**\n", + "Date: **Oct 21, 2019**\n", "\n", "Copyright 1999-2019, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license\n", "\n", @@ -403,14 +403,16 @@ "## Basic ideas of the Principal Component Analysis (PCA)\n", "\n", "We have a data set defined by a design/feature matrix $\\boldsymbol{X}$ (see below for its definition) \n", - "* So each data point is determined by $p$ extrinsic (measurement) variables\n", + "* Each data point is determined by $p$ extrinsic (measurement) variables\n", "\n", "* We may want to ask the following question: Are there fewer intrinsic variables (say $d << p$) that still approximately describe the data?\n", "\n", - "* If so, these intrinsic variables may tell us something important and finding these intrinsic variables is what dimension reduction methods do\n", + "* If so, these intrinsic variables may tell us something important and finding these intrinsic variables is what dimension reduction methods do. \n", "\n", "## Introducing the Covariance and Correlation functions\n", "\n", + "Before we discuss the PCA theorem, we need to remind ourselves about the definition of the covariance and the correlation function.\n", + "\n", "Suppose we have defined two vectors\n", "$\\hat{x}$ and $\\hat{y}$ with $n$ elements each. The covariance matrix $\\boldsymbol{C}$ is defined as" ] @@ -420,8 +422,8 @@ "metadata": {}, "source": [ "$$\n", - "\\boldsymbol{C}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} cov[\\boldsymbol{x},\\boldsymbol{x}] & cov[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", - " cov[\\boldsymbol{y},\\boldsymbol{x}] & cov[\\boldsymbol{y},\\boldsymbol{y}] \\\\\n", + "\\boldsymbol{C}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} \\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{x}] & \\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", + " \\mathrm{cov}[\\boldsymbol{y},\\boldsymbol{x}] & \\mathrm{cov}[\\boldsymbol{y},\\boldsymbol{y}] \\\\\n", " \\end{bmatrix},\n", "$$" ] @@ -438,7 +440,7 @@ "metadata": {}, "source": [ "$$\n", - "cov[\\boldsymbol{x},\\boldsymbol{y}] =\\frac{1}{n} \\sum_{i=0}^{n-1}(x_i- \\overline{x})(y_i- \\overline{y}).\n", + "\\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}] =\\frac{1}{n} \\sum_{i=0}^{n-1}(x_i- \\overline{x})(y_i- \\overline{y}).\n", "$$" ] }, @@ -446,7 +448,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "With this definition and recalling that the variance is" + "With this definition and recalling that the variance is defined as" ] }, { @@ -454,7 +456,7 @@ "metadata": {}, "source": [ "$$\n", - "var[\\boldsymbol{x}]=\\frac{1}{n} \\sum_{i=0}^{n-1}(x_i- \\overline{x})^2,\n", + "\\mathrm{var}[\\boldsymbol{x}]=\\frac{1}{n} \\sum_{i=0}^{n-1}(x_i- \\overline{x})^2,\n", "$$" ] }, @@ -462,7 +464,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "we can rewrite the covariance matrix in this case as" + "we can rewrite the covariance matrix as" ] }, { @@ -470,8 +472,8 @@ "metadata": {}, "source": [ "$$\n", - "\\boldsymbol{C}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} var[\\boldsymbol{x}] & cov[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", - " cov[\\boldsymbol{x},\\boldsymbol{y}] & var[\\boldsymbol{y}] \\\\\n", + "\\boldsymbol{C}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} \\mathrm{var}[\\boldsymbol{x}] & \\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", + " \\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}] & \\mathrm{var}[\\boldsymbol{y}] \\\\\n", " \\end{bmatrix},\n", "$$" ] @@ -492,7 +494,7 @@ "metadata": {}, "source": [ "$$\n", - "corr[\\boldsymbol{x},\\boldsymbol{y}]=\\frac{cov[\\boldsymbol{x},\\boldsymbol{y}]}{\\sqrt{var[\\boldsymbol{x}]\\var[\\boldsymbol{y}]}}.\n", + "\\mathrm{corr}[\\boldsymbol{x},\\boldsymbol{y}]=\\frac{\\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}]}{\\sqrt{\\mathrm{var}[\\boldsymbol{x}] \\mathrm{var}[\\boldsymbol{y}]}}.\n", "$$" ] }, @@ -500,7 +502,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The correlation function is then given by values $corr[\\boldsymbol{x},\\boldsymbol{y}]\n", + "The correlation function is then given by values $\\mathrm{corr}[\\boldsymbol{x},\\boldsymbol{y}]\n", "\\in [-1,1]$. This avoids eventual problems with too large values. We\n", "can then define the correlation matrix for the two vectors $\\boldsymbol{x}$\n", "and $\\boldsymbol{y}$ as" @@ -511,8 +513,8 @@ "metadata": {}, "source": [ "$$\n", - "\\boldsymbol{K}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} 1 & corr[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", - " corr[\\boldsymbol{y},\\boldsymbol{x}] & 1 \\\\\n", + "\\boldsymbol{K}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} 1 & \\mathrm{corr}[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", + " \\mathrm{corr}[\\boldsymbol{y},\\boldsymbol{x}] & 1 \\\\\n", " \\end{bmatrix},\n", "$$" ] @@ -591,12 +593,12 @@ "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x}] = \\begin{bmatrix}\n", - "var[\\boldsymbol{x}_0] & cov[\\boldsymbol{x}_0,\\boldsymbol{x}_1] & cov[\\boldsymbol{x}_0,\\boldsymbol{x}_2] & \\dots & \\dots & cov[\\boldsymbol{x}_0,\\boldsymbol{x}_{p-1}]\\\\\n", - "cov[\\boldsymbol{x}_1,\\boldsymbol{x}_0] & var[\\boldsymbol{x}_1] & cov[\\boldsymbol{x}_1,\\boldsymbol{x}_2] & \\dots & \\dots & cov[\\boldsymbol{x}_1,\\boldsymbol{x}_{p-1}]\\\\\n", - "cov[\\boldsymbol{x}_2,\\boldsymbol{x}_0] & cov[\\boldsymbol{x}_2,\\boldsymbol{x}_1] & var[\\boldsymbol{x}_2] & \\dots & \\dots & cov[\\boldsymbol{x}_2,\\boldsymbol{x}_{p-1}]\\\\\n", + "\\mathrm{var}[\\boldsymbol{x}_0] & \\mathrm{cov}[\\boldsymbol{x}_0,\\boldsymbol{x}_1] & \\mathrm{cov}[\\boldsymbol{x}_0,\\boldsymbol{x}_2] & \\dots & \\dots & \\mathrm{cov}[\\boldsymbol{x}_0,\\boldsymbol{x}_{p-1}]\\\\\n", + "\\mathrm{cov}[\\boldsymbol{x}_1,\\boldsymbol{x}_0] & \\mathrm{var}[\\boldsymbol{x}_1] & \\mathrm{cov}[\\boldsymbol{x}_1,\\boldsymbol{x}_2] & \\dots & \\dots & \\mathrm{cov}[\\boldsymbol{x}_1,\\boldsymbol{x}_{p-1}]\\\\\n", + "\\mathrm{cov}[\\boldsymbol{x}_2,\\boldsymbol{x}_0] & \\mathrm{cov}[\\boldsymbol{x}_2,\\boldsymbol{x}_1] & \\mathrm{var}[\\boldsymbol{x}_2] & \\dots & \\dots & \\mathrm{cov}[\\boldsymbol{x}_2,\\boldsymbol{x}_{p-1}]\\\\\n", "\\dots & \\dots & \\dots & \\dots & \\dots & \\dots \\\\\n", "\\dots & \\dots & \\dots & \\dots & \\dots & \\dots \\\\\n", - "cov[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_0] & cov[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_1] & cov[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_{2}] & \\dots & \\dots & var[\\boldsymbol{x}_{p-1}]\\\\\n", + "\\mathrm{cov}[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_0] & \\mathrm{cov}[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_1] & \\mathrm{cov}[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_{2}] & \\dots & \\dots & \\mathrm{var}[\\boldsymbol{x}_{p-1}]\\\\\n", "\\end{bmatrix},\n", "$$" ] @@ -614,12 +616,12 @@ "source": [ "$$\n", "\\boldsymbol{K}[\\boldsymbol{x}] = \\begin{bmatrix}\n", - "1 & corr[\\boldsymbol{x}_0,\\boldsymbol{x}_1] & corr[\\boldsymbol{x}_0,\\boldsymbol{x}_2] & \\dots & \\dots & corr[\\boldsymbol{x}_0,\\boldsymbol{x}_{p-1}]\\\\\n", - "corr[\\boldsymbol{x}_1,\\boldsymbol{x}_0] & 1 & corr[\\boldsymbol{x}_1,\\boldsymbol{x}_2] & \\dots & \\dots & corr[\\boldsymbol{x}_1,\\boldsymbol{x}_{p-1}]\\\\\n", - "corr[\\boldsymbol{x}_2,\\boldsymbol{x}_0] & corr[\\boldsymbol{x}_2,\\boldsymbol{x}_1] & 1 & \\dots & \\dots & corr[\\boldsymbol{x}_2,\\boldsymbol{x}_{p-1}]\\\\\n", + "1 & \\mathrm{corr}[\\boldsymbol{x}_0,\\boldsymbol{x}_1] & \\mathrm{corr}[\\boldsymbol{x}_0,\\boldsymbol{x}_2] & \\dots & \\dots & \\mathrm{corr}[\\boldsymbol{x}_0,\\boldsymbol{x}_{p-1}]\\\\\n", + "\\mathrm{corr}[\\boldsymbol{x}_1,\\boldsymbol{x}_0] & 1 & \\mathrm{corr}[\\boldsymbol{x}_1,\\boldsymbol{x}_2] & \\dots & \\dots & \\mathrm{corr}[\\boldsymbol{x}_1,\\boldsymbol{x}_{p-1}]\\\\\n", + "\\mathrm{corr}[\\boldsymbol{x}_2,\\boldsymbol{x}_0] & \\mathrm{corr}[\\boldsymbol{x}_2,\\boldsymbol{x}_1] & 1 & \\dots & \\dots & \\mathrm{corr}[\\boldsymbol{x}_2,\\boldsymbol{x}_{p-1}]\\\\\n", "\\dots & \\dots & \\dots & \\dots & \\dots & \\dots \\\\\n", "\\dots & \\dots & \\dots & \\dots & \\dots & \\dots \\\\\n", - "corr[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_0] & corr[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_1] & corr[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_{2}] & \\dots & \\dots & 1\\\\\n", + "\\mathrm{corr}[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_0] & \\mathrm{corr}[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_1] & \\mathrm{corr}[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_{2}] & \\dots & \\dots & 1\\\\\n", "\\end{bmatrix},\n", "$$" ] diff --git a/doc/pub/DimRed/ipynb/ipynb-DimRed-src.tar.gz b/doc/pub/DimRed/ipynb/ipynb-DimRed-src.tar.gz index 359365f9f6c231d9d10f5ea769acba5b5d717deb..b05e978048943b68e918954d671dc43f47e22012 100644 GIT binary patch literal 191 zcmV;w06_mAiwFR^lC50;1MSaC3WG2Z24L5oVoo55F}g13!bPD-FAzzrX=`GW2=(^$ z0qv@#8!3f+o1bBZVWui}`M!wU-36OL2&G8^b7m=B62)u1z?cFiloEn*oW+1JAru9G zwma!%u#TDOl=du?73%HWG}bg9_N=eKGylY~l@`jw=vu8nX(wu}FObWw5Tso;0yN12dC&M}$g};8r^E}V{+5;P0__hED005K9Sv~*& literal 191 zcmV;w06_mAiwFQ<&8%Gj1MSaC3c@fD2H>uHia9|^Oj6fEyXZoY;ssKY+Ne!xl7hXx zeSoeMH${Yeo1bBZVW!M?**;JF-AAiI2&I(5l-rcei8f0;!>i>3uLM-rASma=n909 tZfb#*w_aHZTo1s7P+l3$uW@J1$@11E@vol=f*=UK_5fD%6)yk?004H;SzQ1C diff --git a/doc/pub/DimRed/pdf/DimRed-minted.pdf b/doc/pub/DimRed/pdf/DimRed-minted.pdf index 965c1920a27a96fda0fed1bb278b317793375389..a46afdad548d351886c2e4d285f6e19a8503098d 100644 GIT binary patch delta 12848 zcmaiaLvSVxv}|nKwrv{|b7I@c1QUF*ZR?9|+nCt4Z727CukQXXPSsg-^{(r5b)SY# zq`EGocuG*NBqbU8dX*zkKvQ8>I>&rOg%1d+u7;j@#x``gtkI7^+>$I;U5E8=iHB)4 z6)AAax%t|d*nHM+U26KV%O)e+o1Eqa+6SbntJMRtjETYoGl6;G3mfdj?_74*q#>$m zCu#`4^TyBmW+j(9^yVZSmiS%d^&%>!Xq^nUDD;V_+y+!AY-C$);01BrdK;#NCY|v_0CVM1U?(9es{P#Tc-ZjM7Q-K(t;jxOpNg()=SR&GK;01Vkg1aHSwDMefb;xuY3CS(GofGQ*Af3an4Z@8GAJM1ypc&F zEHu#Igs!49;yU5Ol%9)5Dt_=GJ6c|xoj+r3kFI$`)o0F$$b2s^LYDUT@3LQ8QK9z$ z`Y!8BtO+wcEo@13rG^9A*F>vVnX>VQFQWte79`kw|2{+%0D8afdvT$^pN5qRhkE0k zA{~f~tNT z54{bR+y!aAW@{(^noFt4(QWHlK@dU1;ifJg8ruY!*E5L<(pJW>mNWq()+qnh1i9Wt zGpU52UwW4n;OWmhs{umv3YgTz6+lzTz`8^amPVwEkJK3w@!hCD$1WU_M|8s)v@8UL z2!15__p&!3O+J#Qm3-dg@bwn8#$QPYCG{`xt7xy0QzDNC9PmfgfUi2YW_UKAVC4$sph46Dwd=MF z-is%RVJdj8#y}SFkDXz$)kk#MyIl6n-Fm{z=YR{c(5cmr4FpXF`p@^-HRH#lb3sUo zk{?mAEhB#1(1^_yeUugQN!#-p?``r)dCx`4OvWaApA%aWuPK<{Qp7kyMGzZT%bmRt z+GgWVfG21lb%yLX^-OFxi{&R^BldvrI=Jzfz63`6(v;YCr~#nCclvrQ8aLb0 zGcrtuYd;*17=MXahb%a}mzB}@y@3@Si%4JBLqTf_6rP$=FyU1J4T&~C@(i0E6>*nP zYJzHAl^{8b&u2!Gg0}eMJ+rBK%;G6GGrNQDPWy<{@XGNZ!}FdRxe({ z1Wh8y{1L+Pq#5`t@$V&*!3}(kJkr{5mT`4Li_o}sRiG&iP2APZN|ZsFgb4|TCgxWB zDYvR@CEfC3Vxr>UH$}4Pj%G>V*elpP6usB@L8<#TEM-^;gzR%Y;aU*R+6|{upwGaxH=lYr}QbmTW;7iXV zT2>Seh_%DZ%DwICCI3vm_M>sGOuAa$@O#ol*PKF#!^jf?OzDSsTh4`$dWfp>8&X}Z zIC%Ixs}qa0`nZlzG6q=oCPeBdT-%`^pk}eFHL;tIBnqLVJ5s|a0@v*{?+l{b{OfD!F#pT$s-RJ|0?NqGPO;5M7)hU4lM}WWuv~)P6pcs&AxWhf01SEh zeY16M^FwIXEq6ZoQ|c^DCG1y^f1t+SnQNP!Aw^K6HjV)2R)lOrzi;amqUZu8XI1a) zMFVW#=p?>XNgr3zLza#yU6%X?I>i{+f>GzJRD>RR?vGLo58Tf!??};Xo*wEctd6QfTm=O4a6N#k=baD7B!#iXWBjX3e~-8@5aWHO&oVK z-d!f)tw8CkI?2xOl*^N8Zw`X;Ij0hkEOgL*$5Z=eGV`4t{~9+v(orSm+d+T0QT$g~ zY-V*SMfn5S(i#4WQwirp6lc#M){W&47Uf!!F*DCt;tmyWSec!cseiDefZ_fV%u{QF zmKm|XKJ*|vn=OHXrtG08xN@c%5`f=Kqxn8RZcV>i+ER%^N;^mT*~{n~0%xz^rmU8! zrtfc~2W=}#`HzZ|8w9e`7oq?*H(`_fk<1P|AzLerw!I(A^pkQwErsRcp#B81=1$#x zWa}N7wx`vrpSnD88w?%&0H1sM@uN}I+Q>FK5VTR_Ofcq1yO3PgQE}bJ_%RVvFYVsB zO=!nNo=>i4%R;u^TZuX`^R*wW(u#`pStYt)1exA1M?k)k-v{|2v)g0X6~ps~Qp%H1J`9kccQS%@jpuVit=CXqxi z+lW|fee10)4ZD}xURjcK{KMBJ(m6K7}j%5bg zCYWkblNXSbxIw)QkowBg5tx5`@V5Wc;fobeVG=hQj+JfaaiODi6Pb^4YDgLqW|d(vltZ~0-n4QRZHEX7XC2rH zopKfQG{WBi4i1%ay{{qn67u5Yq|tGl9KNh>4aO#gYfVo~4J@FP7f()wr}d9kHdVv= z6;L1*ejQW8{2i7i&R7H?Ord;F#9 zilAK!p9bbv3frMyyV2%r1-kls(=U*l+FC*s__kbsoR0z)i;rqr=K;BzUL*ZAc8l$v z+0*PMQ|f>dbu`mI@f*4l%0{K--vK_#W@XOCRWQfDvGd!k`u-IbJt@z#m}3`T@;hy^ zH6bY!cDr81a6?o0A9iuqPgXe0RaNyy-;Td*)C~me}QXlJ@jj^^X^} z>IjG{{4K4Ljl3KRrx4rupxdbU|Kj|)Auq*T>s^}MZv;FuQM`BE(zPxRwEvXhYxzBU zbio1uC;heRVJ$n3(@8K6eXM89g0jj*abWMA6h3z5e`R0_hiuQhV4Y=ga3!^Q;?3m1 z$iZph#5rV;>tMTdlJ}LuHcr*LV9x^>3It)Hjm%CTIi<+AbN1g^P>vneD`NQ|hcfQ& zFlsG3D=L z&Lg-U#!6CU8R2FC3u5rWz_^t*wl;GoL510fK_f5|{_ytV1{7D~dX;dUvRPjzN2-27 zot3#;#MLk-TG-Pkf`e$~wYN$kn=?c>#IB5j>hREAhbfBjCR}?K)rtV4PPMrYTs}ZF zP6hP{hkLmMT?&;CySOMy|GyzPDFK>+UtoL5h=D%UB%izP#?gWW zR18rLAt(dH7>k`d+Y8&o=4Q$#77EcL@5Vu%d$5sH$W22_6_y!v`IRlg>6<(QV+<#%}djD(Gu0G_)FE~71T*l|b@*0HL#|NOt*R{IKyy!el4&7%Eqa}oRp-~0 z$rw77UHYncKB+WA3>#s1RsldbJz0WNlWmnV;G&N=lbpZWZ1cG-m0vDd#y0jqv2dZ- zU;5CW%BFPLUOLn$+9#QSh!hIajnOxoAl<0+a5e!CX5V zJ3T!7)m4kk&M~rNW`;_oSEgSejKVZ2&0MAo3igep^_vZaA6;1CatDAGt7Q$7z}gP7 z!;YK~8UD*R204p_z(9pD7CakrPIP;mV;SO94TVWX)juVJ57GJl2HN>^jYLrhYyXfp z{9w-}YE&ef$E$U&7hat`9iBo*slP2&9crhxuMIaKv=T8lVdR`4D*>a;6Eb1kXxRs5 z5a$;MDn#L}I3-KTsTV*isl77FU5A@L$%arVih<+m$5sa-HTTZh00Z^PYc^WWRS1|M zq5Gv}#pK#%vhd^m}439}CagwA8Nq2kDJfLu^{* zbxV>s`qYYucLA7qt}}}~+x_%Sovrel!l}FTKV$WReZRSVj{rMOG&^TlzKHHM4N{cg zc^DV2;@Wu-j(Jf&s$2FIoS~uC19K)f3?giC#jh|53*;1N>3=}?z8-63!!sbEpy&2? zkS2SgZP;M+hXUDx!kx)VR8Ue`IgxB$+se>sWy6i9c4bA~8%T$e2(B4Xf(VsXz{|8= zcw+Z~mVGkt5aVY6N?aakB#iH;S>Xti`eW1gy0@cokAr_=D8XKtV0#km2S7}c>mLvn z8@0=%Uqsa4JlvfB1rIcUD}oJ&jfp!!!vbkIp7qP=ZQIA{uTyYzg<=u`-E2vl-5V0wx*H`F&t7c*yWBPp^D%U-!mCmVKv?t8@6&ZVhu+KtzX zXIDI(>jq}ES=qN&YdEU6bP?h;*sFVVY(Nq)=wPHBld;Qw40RMlMjxSf z)@;c>3yA~NL{e(9Dzg$dpS*mi{|Y{Fq*Ag?n$;)NEHc%XkXN|>uC8{C)aOFcr#lR z2HD;=sY7$St?Og?o%i@!VaA1z*%RBAughzz2K@ufu#acJkw9Lm6bnd@CTKFsPZ}Jf>4D8C|NNZ@4&}Nl$EAS@<`+Nf7uoq%y^k&&_oZv=L z97*LIR?#vT9N&BrvIkHU`%AMD*$6(@M!Fe%RticK*`>z%#e;gcY+msYcsx+a7>nY^ z+IJBMIU36=$X_#H&@X$eaK;L?00autgd7nUYcmzVp*noSIvj4uI9r&J{Xz{Hfgre3 zV*e)MHN|ZsJi!WOr`zomRHNRy^nbA_R*PzT1-0g%`$IhzyQVW z1KUS!BThNs-J@qr^WifK@NUVsWA}9h_-+McoowTtTRFof!bnR)w`gq&yiw|ILTZlZ zkgkdEWuAZNug6LT4eEJ|&TT|yuDTlnEz~F|DM#*Ik^oU_Mv4?+#`=f|;i_X$JcOlP ztiRQMgVS;7Dq?9QB+HagwH0gP?I3p9OXDuWMBc_!*d_A-ao0-X2r6fXXm+SfdQbo; zMizm&p|s}t>WUL*$WsahuHj;;GdQSRW!dw%jm-w9Em!s@8fJJTKDDFP5Cn(x)~`h^ zio_fy7#wyS99778cX+cR2zZjob$_hBS#ns5za-n?K^BQC)A*3u8W%4k;)Nkb!gP>B zVzns4Ii>YE-$4-2yhji^Lg8wYVTJ)xcZF7p9$W?(K=&UI4(fwp?9gKn=E$UneXB9% z1nj*+PgJHz?7hw-Nly&}iBZ(8T_|>U74^kft0!!6$yUcorSv|Xb?c{5YOxHf+*~4Z z^>dw3jbPP6o=JfK)fvm;^z=PhSrR?eAb?Qv|d{p%wuVP zx_ajZ^@uNSAC>d5afU zIEAuky2Erc-t++I;8DWKxwwx6{}a!0!B`Z{i`EN>Y{#z~LT$daj>B}b0m@MlSl_5a zZC0AEOv{ZX&ZR<`ilu|h`tXW;tCBwy%&CxYjtiec63hB9ZJy0r$#)N5ii=5R!;@C4 z3#(#E|3I`D+kbwGDhU7(X*RGDHNk>qHI?86rJhRU9~YGjI7%3vxzw`FnA{)rd|SG1 zC?;kN+>mrpO`Odfmld(sM8k;+*P~Q?@pMDZ)gx@kh(w7Q!A*zz>&!wtTVxx^ATKkR zak5n(NPGVZG0|~G*Dm%g3+j%x;m*0uW%6;qEFkbV`j0(rKaKw*g>M7j%J zab#x+ZH;s3V77M0_h#*|Ol7lSwtm=y5tZ8U2zoPpI@&Oqfqqoi`8J^s2B(Md!}nbh zxZf1@3zV4qBil_j6+G-b>YiB-wp&lD!2$*{i{?yUZExi{W9a>l1B^c+aGQqZ&R8i^3*g0LVkEv4$XCHge&L9DkDG*- zcjI#iuMjl9;Z{-chMGFZqJltaB#!k46;WV>r8hcq|4@)tX#8evH*i1idR`;5(}nnD z_7C%Ccu&(=&vrDLauuvb9?*UfR647C(y8gf4Vxqi6CZ%nu}{_SyQU#pX3l3gl(cDt zN`d#h>D@ccCpq0Y{il(!4v%B;Pn-8YF=y^z{M=s8Ki5^Sikd!JN@*+d6*iOv&~#!k48T0}#ITx*#+ zp(Vm>en$X&(=3EkC(K6puI9-uNZCq~{`G;EELeFrlHHkv3}sU1x5M@Ny+~ldJTsT_ z3Q3+ytkDxm}Aw-zra``ptU zjr{ItwPWZqv3?9!7Z!dL{ZWUil5d;p0A;71o-u%+234adqCRC-aPM(NufqAAUqoIH z?Vsr9A+EJokEu1a>c%?C)*5oD@LbiY`|p!+*)fB7!n`7jKqVAR{upQ~o}|@0+2NeD zX_x5L3;C44UztUptVIUImN-mu=QCzbduTaP>{e~S7{M~Kd9O1QWZ|mSR$C&G$#|L? z>j1!Xj|{EEi&I#2*G5`UXuXQ$<)q1nK0y)_Y(AFtmKJg3Agn;io^n6!1G?Zp1|?wf zAr2%1eX+Xo4@f@`Ws^5vYtRJ%8p8JaHh=e9u@aEhiHzlf!$Ji~L*nVlk08 za$>i-oP{N_JL|!Qp!nTPck>bZ&`cIPDn*JcLBKnzyA;mD_Cncg(ruVR<_cG=gw`=< zn$7FG9BxP;?dDjnQHz`VUL)oUo<}@5$vUAubB&NfP0j_zNpd7v(EsY#y5|B^bOS6p zkJ_Tus`Txh!-0BNH-D5pC6Z?fS1s1>6ko%XCz+CAmOgc#wKYx{Z&1ZOyiN6W%rx(~ z{M(*rv50sDp60E>CpV+V-*cK`D+ygs^}(2w31UdI{fo6f3Y?P(5%-q=_*e$lJr@Hx zhiHK)NG5(jX0AoK@&OI=vy&o)uu|F&-WO>KTtJb_VPbpWdkaTbK983AY7lH!8>-_N zmRwZ6>-fMM`9jW%0zbQyJ+)A*jMGks49K{9w>n(M_*8+ybXgHl4?*! z(KK9(!Dk&{ZJG@;0P)LvuW)ltM2J3Y8Ky%8J!WG8&-0D=`oZyGm{% zeOcN!Y9d~}6gyn99)#D4mk|t^vmj;Uebm36V1)FTj;sm~{T@6#Eh=#PhbRx!4b-Kg z30zenQ-mM}c8{djUCt9SMGPfT0kPxW$ z5D4D@#B1HX5gtQR7(h8dS;oMB$Z9OCIyzQUL?b8`#(xx|8{I$dzpGy5VI^WBkSm+2`e6%-V{PvpVnXm zLy;5$A@@6hOq1rgreCwaypN%Xu-MvO+3v3n(Adtx)SiMnUt?Cq+9WxjQZLZT7y}ZY z1}E{ab^$5FUx}>3{!~xpxl4(*7-u+g4u#O@ra*=?5{%iPDB}q_7nZsvzS)*j?RCr{ zWpJ*Na*$IQL;$?-_S?sfq$@=xb<#nE6;Z?s5vUtY=GWY3(Cer$OZNf+pw~+B!Q{1l z%let><=C2kXg2f!)Y3Bs^KLi4GCE`PSz-L>fO zJ(EqIOXej`7a{}disO9zSLuD_M2eNJQRReHO@2S3mSrFh*&F<#rk1|a>k5mGdzN(F z^vjAW2Fz>mMIc=m6mD&8O1!+F`=$WPI9SGHeXXiBg8cNV5x@!6nRN&_!86_y==fpS zwo6MFv93MgVq=<4wYGzkX4;7ByVk~#M|P&}sUBN=(wek$_l2H-%__*z3c`r5a7JcM zov>V1B*Pjb{d-1Nn4~cWgB>w-y}-H4(78)OK#au){*-|+j5=EoNVi`qDV>q6_~P!0 z)S17bv{DYB^>A}t1O~3uZ%Aai27m0V6L&0Q8|pkUwG7V2zoQ77kuE+!sT{ld*`cPrN`RYhoRDzv;?L;qee6^ExnX?yH}sW86H{?OatX0L zARN3_Ls$nJdcDZ6i3*_*zS}D10o?NAEEIpBFYISTcK&XvSa^FbE^v+JLW5a2(q$?x z@cK$xf8Hc5d_bqzpsDQ#sp@8Q#lQBhfVg@k2;<{!i4XB|J!ek{v8R}!t;3=-OqIle zN307-KJ*=n@~1cHnA-de$zRLjGpkPzz$wOoohPjSOBjH0lh(i)w_XcOdcGZJ+Ns`? z`3rKoJqjT`hzMG)=0t3&`6h}|jF{h9SWLoNTs<1I=X~N6=L{3uW~G*|u?ckmUD2fY zVCO)e`v>32xC)TcMb0u#jvUWuJ$F^eC)|S zx=T8)WfpsTQqXgBQ))z&){mx+z0O>w{nL3X=@1fDrhS0Sb!(Y!vIsKd+I_{Rh+b2c z7W+4BAbM@SUaj%nNtbm%0%R*(11yQLKQgf1X zK{<`_J)*6-QlJ&j1kfZI&lD!_1yL@77g%`}IiwU;iNqj=&T=I$!k~6M;jBwg4fM}d zJru!K&>{_p5EK*6{#JY>(uO3#=lzQyi<4|j4h@AF?OdsBM_2cc31W53o+T3iTu1TW z4ruYmad6sHp3>WCXl=XS0Q^QS+0+y4Sasx7MXk-fxcoy|j#Q(Jrr1}E1f`Xrk~M?Y z4R@W}`zNsswboTpB?&Dg+HJ&&=qWUtEUXH63(w0Sq7^8KDrs@ke#acTjpYvJfO)`YBSnD(-~xPYH`paOTiw zgQ#XBOG=_lU~Hn33wVVV1INS{0i^J}g;0CUy6d}Mv!ZI`rRwt$R=X_V50pS2J||9b zUd#*!VOhE}?{`JdoO_1)y(cJtoOeWvKZb0|FSP|ryeL;>Tw-n$n<*3Qmck-7Ui{7j zs(Kehhl>uQqi85vE=^$jO2kGU!9v#A1wf!g3obcC@tb+bO(mbH-P028PG-;K^eJQP z3t>L4DMQIYT#i7sa#;$%CpIY)uSe&!WKrG&gE~q(<0VuKd>jfn`!yC;zrUouXI<0( z7^q~Rb#uy74$-`ud)6j7{I501W{mv-FLZup#4Hm@#>lRpGLYtBjRX9Li2 z31a7SAZ`g-Yq!eXN!IwR{*z#3N_19MXX_lIU6?PXb^4b`lM8Q1_v%6^!bupTM#b_t z>a9v4Q&B@mjU)|F0mnJJ0n^;9=3K98W>vmZTd9j>@{b^4Q64^Ol`Osy``Ky8-1t}P zK)B;K0W*n(x$+chhwgk@tlF+taJ`nw!nUR4vVZU^(MUX!W?1S3nJfYUN55MAOu7Om z^~FKi>39mCKE)CBH=ILI(PP(q`{TWFasmT+)a2*>cE2xxu#G{5KqAT>o=5;rE$!rR z{~ohHGWGZYao}EoRM&ztO{I|Ns{_r*&Fr?`c))0mPgCGTY4Yo@zzezUmt!p1(heC;GBgz$zs8%H;ku?`Y{hj3C13_2Ty%7d@EO0u%#z)NX7JjIOJ zFO;_mOrJvlJQp?eU<6)%NZHzF>Y0^kaTn9p<~vTRY9}V%8oiy))Ne(?^*h?JyhzhA zyqcU<%qO%EflLJN%?Y|L2idHXYM?$tZ5=&b?j+2SpY@r-@c9*0_ow&bv|dh?E z5b9eoIAeH=^Q9>MF}n#anKW}ecz1fZ_jckOg^EBhwb7`R%FM?8`f?$qd&o;tAT^;#FO^|SZ0MataC!k#jAT|~%= zo04^1Z<;zemVY?<7*2cc!sYzg`E11Hww&@}w`R(O|2rk%AUx&jD2LC1edWf%YhN-| zmBF1hr4?B7=7HVihJU+#)ywDic7Z0~OXvv*=mRoawfF6Yz;(496SWX~ve`r47&pR0 zkD~~I2G8IQ2*U-BPSLk1LfxCOso%Qy%1X6Ngi9rVAsbRSM;ZWP ztMcuT#)jZ_>WH_aXSauWSK0ZmJnvo4tG>A_N>{D#)_(P#CKU9FQ76p-UMJ4O(Zv1s zd;)|y2%|$*?yyPIpzCK=?yN-aNfO_UN`|IgzR!$Zx z#F<-l4v={sWRHf(+b-L3rZoLJ;l2PNd45}C*>AdzzM@`VqmzTcQ9|W`r}Q$gh{BD3 z-6Cj@nliEQ1mq-6a~cnt&_mk~gJ0cgOQ6oxk;^iXB5)$_SsC*=94mt4CcW(YnIu(> zn+xWtAKeqK%)ATTM|xVr`qXw~u2Kt0lZ{HI5Q~3>4r8*OFO$1UzAjvNN-zMj!oW~K z51FNjoeS@lJ4sI5w65nK#MN_BgpqdJ+3>4NXx00QIX@6Y%972IozmOs1C#%s)J|%T zzz>`KhQ)U+Y{Sjbn>Bw<&aaXyI#li-hti}P^v&~p?~jkl6~|PKz}{_W0&KCzVl&4z z=uPAg5e{8nf8~&U!lTFP*i3*HpvUL^eORm=-(*!ve8M{WAh&ne9-!S`x7FX+TOTP8 z@0hN-LzhO}_4oU8W-FxS4$3}c^e{qwX?~TPK2N40^oVI#-rHOH7H+SmuQ7tV++&e2F6+m2g+AY#dUs zh!(U)i|Omtcl`AReR8+&|2ao3pr=(BOCo=2=b|6 z-k}QNdm46_TDi3^M%A?GzM;Fl7M!%Qd9N9_b+NaOSnI9J@FIwMFftsgn0daxqNIS0 z-7@ujyn&X53wV-w2n%oFwaAu~!?k*Mm1i&PC8#ng!REoWihMh!cgd!x`xqy2Z9w~* zQM3DyQzw<8!Nx19ljtRUtm$y)G*r zG?)O$F8Z$02IvZoB2oi(s0!ObRvVLFf5tyUS?w7!;ONujUPxA-C;sZ!d3GD6)Gi1= zIwV~RLsax}#iZ@0fM?UOva++XadOeKuu-$HP}9OOD>#}-7`vI1&`9vHv9R*8vatOx zB&{qJd;pA_|{px^q(7mNr?!`25z&p->niXgEHun7H^S#ME&6|RTBLh*x>jVpi3j;aN6A#zK;tv*$p3lScE)AWH zi>L=G-R}?o-bf}QOI6UHD4V7@pgNk6mLE`{tv8?C%#nx@3Xl~igpepQQU#HdA}xYY zPc@kbj)UySL6w&#R8Wgd&uH%dcoB8GyC4lXHEm%EwEBJpmW384 zK+n-kC%Hfs&XtbeExk^ix#3~AN()Ds<&>^vHnuh%9^Wx-LudpKQW)CLhmmc7oD?gv zHCN`3jvH$2BYJWuy{zkRxn)=apB&v!FG`8I&A-a`k!xPX_kHH(T@=W;eihk^-9?oL1wV;8{X$~TS0pd>lfU_!J(3T)a?4-&CrYu6Ei*8K4 zro5_ET1x}}>(5&g-tU_!H=#_IX9$HG#=5Jt#g?Y|nH?V=whtbIc#g(PA)+eGRer_9 zKhvFCk zk~_uU?erU?-$84UELqFSd-&v4s9;hRh_S1?#XvB3qDK53AY~=VPKvXJw}E2EKAof?>T!92mae-q z0u%Knb)P(7xpBR5cQJQGC5y5Y^{ayB3SSn_+{d(BDK51|Ynog2TIaoqFgEr9gsr?J zg?Kw*PP!z0^F$<3^~0J(w)~4DKGn80Z#7i^3g*lhUg?xs1Z!S5pSHI+-+bbJ=N7gPTDq-Zvms)BM>j#+ zzAUk4_?(@v=M$auOU?6LHQ`T}9G!}VxRSZqf><7B9SjSW$m8$%>qv)OkB+_HrBi7X zS>Vp9QktJCqy5P`IotarJEHA zJ1@`wQ#}oUaK2MjO$B4*u6dd({(urCn8F|s4h)Pn@-CJ&orVI95)O|fA|GSX%GB8S zfX5p`o{@m&5Z4o&*nb<>me`-@-f+sXr*UPQm)}(gHDOSM!Ug(zKoI4RVGo=vX$6`7 zGmd|dhPL&AxX~`Ks=8m72MYDnU(s_^X!+BbsFM_+4qN&BQ02AZ0j4-W8!n2i=Iup1 zr13Y0@FHO0Hk8r*!~jjrafJsQHs!G-M@B5NT>pwJTLV{N{gEI$jN@?-5+_KWu08q+ z-(R#1L7WAF+zC=@4=TFy5a`YTu0;yJvdj0&iS*;7m9$iKyB?i;4{I}w>em39fIrmX z9w`POA5!fWCN&?1(i1_+2+txP0W)PixBI1h;-jkgcS`R^YDGm#V6kPz(@kn_rNbe^ zRJq@`Re6c%YSW`w&^)m`Gw14hGe>QIKR_P-98XfdcCq@W@lj_hEWNm+LurSBvErlM zo;X>@(`K(cq*X_mf*5*Ymo}&sP8l-~q^BKOsrXtD1J_UpC1m{d55u|NUIt2&1iE^{j!m7W4iMCd@PqOtOqa^izc@tG7*%`SnHAbA| zP!`7gkS#1R#3uxpezYlS#BW#+tnTvLmi;;*)Y@tolVD4cAfNCm2Cl9!QG5zoo=`<9 z=YNJ|{oFu6Y>0=4ZmfQ5hc^(eN5r%u3p1OH( z!dYn)6B)LNbLk&k%;C|g!4!MH2_w18Ogeo~ugy5RI#Ya?$_Yz5i16@-=B@Gki7fZy z#9kPdJ&wjH((a!1FDrn(UniVJEcYe_8$B#nFB==XKUkByv_9QPn_x9n57uD-Fq0Pg z5lD#8J+ta+p6EN|FJl^Z$}uG2r@XjnVHO^2O+)H7Z6&XT`vQ}L!stn=jg86~o!E%S z-x^*^>x{`0?X^r%<+)~KzwRg(?h=%f%-;t_x$OwiM8JdaNvMFqZI5yi;GdQAg;wPj zhdF9s@w2?2_D5)$pUVRK{daKc^Q zxPjD8=#obS#rj=+fF1i>tG%<%>ntCf%I#faQe>uKB9}WV1^CtYUIQ^QID!F@{ShLQ z<7z4e-r%@?(>oxTYkn1&z}F=aN}Oa2w=1Iqru^dt5mw3p%_bfFNA7Id1P_Ut0UjIUp=d!eWV8GM;#-w_ zam#)>eW8f!G-``|nMDQ_uDt0jc0J`CW>(PIm^Idk16}CXF;72%lXxX+i zyX)#@WRwDyodvXU*mGa>@A`A;DISyT`t=HK#k%vsw#uyW_BcghZ?~i$SP%34iN`(jYDR&9-lt-Y^oSg#xv>13Vk#o_9z?n3?eaoi z{tVaCtc`oNQ53@P z%A&x>+GoMZTz02>m<3%Wr?f{sWd>qAIjL4C42SJ7=OBZMT;*}?YJQJ$S z)aUZVw`VrNzkr>cI3(~5Uu~GUGEq5&1S7NNc+dj80rXWEJ$_NSxu`%% zmSfwc0NYbU?tMoQsSCye6%F)%{@~D6kGVT5y^%Q(1WwivhYq+reIFl7wSqWec?ZSM zJQY1m|8S_?>!V*!M8}QYvf7M9%+<5;&_&}8xfpT%(j1!g z#$EdL13IDOHjSe_KIu2+a$c->REzfI!MUg*g7@aPY$PUo{lWFm+eKsZ4E~0LW5#md zB{3P4$7H9DCMo85sx3qwP>>$n{V;8!^>)}pLyuV?kg_F@Rz`d^>cs-yzRs&bc(oCF+D>1-yRtKYq9pZpCz}SEnG>r{!JtIvB8hxk_ zDC$7orEq?=znT*_57>`1`aRkB&8VoZ-cpdK28$aX@V@sYlK1f}HEwuzhPUX8tFPN)tNB!Zep{@SdnPmI|5d zYnpT*-ey(Sg4k@MZIsacI;J)|*N+f&YZ^8lJ}RvS!`bFq@1N#FN9-|2uhoeRxXf2M zMyTkAUbs1t=$^9<@Yu0W`@k09IyYI)AzGQMcAjh)cN#ZO-!XHqZEBKrzMAJIl@N^q zMK9ZT+|0m1vH8%_rYgQMUSJnXFXLtvSf;*VTL16bu&uwZmUR@dSPYz!XrHepjOZ4R z2Y%5Y;A|kT29<@j9jF?vZpGZ|AN}G~R%cf{@sH{hSgpg2us85k9K?p_i0lJ$Duqlo zb8(XCI(q6fH=j&~y2gIhkKv3MB9D%lO$K=mZ+wH+H{aSvp}nwu7@*E0t%xO|_%FY0 z;jZbXXXUX^f$v+0R2}4o7#ccDt%F3?*uI=zEVB@2$}!sr8iicX^_p|V66}pF7EkRJ z@SQ??i;=PR(0$)^CWAJT0MF2yl+z7*%CK_l)B_jW{qzHwzu&%0i4&O%?q>@7Aus{D z(701gh>;iS@jc^E(?BU1rm*1c$U81akdG$Rs1X4!I@B^^LvuE7mWQRm+&s=}6;Z_p zQuP)LZER&@2uVmwSGh{gW_gkNtMbVVt!j-#BAYRy!5u~%#j0}gA5$*YpGEh)38opV zi=qdGW{ZDQca8J!w!&~7&F-c%_YQ~E^}MuGR1Bg%$5@`mNP&*sjObIREJpBATBo0l zH&1MRlrBf^I9U7b(p?Gp;GT^6ag(fyp48g_K3|F)%WmxBnCI-nq7-@8C1-27#G6#% z50qS2J#`4l?hxUJ>Ym9>C!!mz)Si?odcL>Xz>AaPGR5&s1%1CelXTCi!=I7ohDxAp zdRPrSZwKNrSKzEF_c1R9!?DVqVG=DgpTZONpcsm%0?|of;W9K1Hl+1b>J0Q z%IANUJ+xJD{=9p9oq`Rs8Q2Okpw!oJmorK6u^%SiavVZ)R(k)>V8akm?!tssiR*>ZWR6 zyhYizz}c&X(A}Kg{9qxRim@)V4MmbdXuaLTJEzW}!zD;B91&`C(f4u7(P%eYv|$~; zMvL!$-n*xYjOvN0ze+}IS^aFc(5TrY7H%k&uCSZgU8_2cp=D!#mlcA4siycIB9GyI z_)F!dxkkuQ$vaxc-8|#`Na(S%cy65rtgWGG0Ouwaz{zv@PSoZKJ=x!-PsG_0N*d^m zMmLfEj!YwETAx#?vJ4KeZ5ZB@wZ{`frsGN2rVcnN=MTBxHkIT8;8ekp>d<*k!P+Vu zPL8zMDU-_jJ}mjm-#yyX(LJQK>ZZSUq6LN}a&kMzH{q+E0dxRKO6eLsd$cTFzNQ$WAFFBe-{Apksa2RvD^oTf{9tu9 zEz+NbdWTF~`e|>|D&v*yg@oBuL*N-p+*q697K`l#~XLJz2u|vk7_JoJh)JL$-mfhGErrVwg*{4fcIeK zoNl&I2NOxX z2}p}m%U~C-IVgt9v_(Vz#Gapab3877`R&#YU{kN^mMAs{Tt~ZJt1l3+7Xr?ysv1YO%fueLaW|kEC{ii>E(xm16LQn<-c-MOl(kde6c=ykj zz-Pcsqss?3p0D9_I#9>X8(d1^^1VET%?jC_O&1~L_+=N!5!;_Va35k7akJ8y%Ib^0!ajm1T+;)P(wSXP z7;x|`7<{iiN+HuXq$xMCMrl}Yd!S2YIJ8;f!2rM_!Nfc;$vWt!&v>p;6J*M0-MH;_ z!+rMy&lhkzTwk#f`xR{f0&1yzYTnQ{M+$1_NBhSFcS`OwP}st_--hboLb*hQE1PnO zW5)dCtmu#l6^x9@`Ds(4eEr^8E0p|J3sieLI}|Is?0$9`_ViX7 zfoX>z7A6bQ-oeTe%UJ0}1n^rtjde4hWQpLn>e!|*8IH;NEj+Go0coEJNLAO^H`TiI zJ1}15_<9Dt^y12%j;RJ5XHjlJ-gNSMpgb$cJ?0QOGsMrS^GQ_p z6#I0CJJ_hkbkjEWMiW|O+9-fK!m!xPz1xUX56?-q!zE#)kB7pEqb)ovf9W*gU6Pz% z%$X~P_u3N-lZCPj0ps81t|>;j+|eFLg$^YLn-j5D%S458*b*X~r@0#=J`J#Hx5oix zUM^IjAwMB`#-L0>(JkqQzXb#_Ej@fQlC4YnfFCSv(1X`=o79F!XUVs>U8cGBAI?Z( zR+!7zXDl&UG)z=JBTwYS#RqUx(O2Ax0SuzV3`oBC=G*;Vz}$;Fk^$~W98#igqWDp5pN%Chdl{oHa!Y~Ub@1qIT+eQ6y#30AR)B?m?Kdv;_)1$?$A|aWv z+vwkr=GWd|$>Mfw1M=B_Yc*IdlS~Z+oWdj-Y^b=(qM4IwQM$TaYHwQ8FFOdeQ86s0uyx&^lHaCI-jL4t}@_ZT$ z*?F1|NXX4VUSb7lz&8!+)a%Y^Qb1CbA8{$)7{2804qAUmY<3i>r2oyhsd~WQjxNa- zU5))I%Rd(lL<12Q6uc{5iW#mw3zuEfhx)4JAp$F$M2bB!XI!}7D=?^mq2<5EMJz|6 zg760Vqd)MGtN(l_`4Cb=a&rCes(7s@6L;8-+51v=r}AUe?q-U^WrBjxOe|0a9NowV ziAYzSaac~GwX5>mpGaa-N1oARf^JRTP!{b|T&OpPxMDCV94`0cnDBW`c(=#*I{_34 zkW}UE=*m;BM5Xp2;{{CCPwconxIVs|GkXI66Evcc4n)vG^q64?^GDeLHKoy0&;{O^ z+#*HDNX-@RNarD?2n_C^3?b#E#XBb80y1XH7Z->L2kNg{P+EW03vEBRb7G_QOsGq8v)YWhRmCp+_*V z!(`56sKC)dujRvAUzdP}xD*aowwo%%ADwNy)L4)hSxUR!es#+caoq+%n6c~;FRr(I zyY?z2oq}oN;M#(DLBs?O(N<84s-&Yx`(0Bia>llXFYhoMa=fu&(mf!U3-5N>QfX8o zl#7mA1CvuY~J7Z@g1JJZ1l+^!WeN?3CvMJy!YtlM>jw z-ns7k_DmNO_8>x6wbV5c5B0FqZEKW$XVn(+`US2MU;cNHE9I#tM%Z|iyQ4J@tI-Xu zM_3kpbCuFsgHY9|n6%KB@|&8(31T3QaE7EUS(ceymL?GmI4y!w+)(YNfryGBdd4dL zqlis1cpNT7i@sZMLEaBXPZ^wE2xX#MMI?@IzVt_NQvB*iwHJwKf3;oy5rlPRRfEjt zT|xS+C{iNH1ukIQN$R&?3atujdr=FH)_EA2hIjyhQ8+vWM^^Hv^+DLnLD=I-7>Dza zVF*6nfg~9ma0OtUeOty(meM8dTacVe94B1Ka)?2zFnjc|fG@5t!}-Dm6CUW-Cf=XhejL9=pbdR=hGe=^Uw@)HIZF+=H987L zI9!w~)RT)Gyf8`9H21$uv%SORmWI|dl z0-gE{qyqy-5Y-HZ7d0xBLFLq584h1lg%Q~UXyO5%;4i-o#O??%p^HcQ@udCR#+zLpabgUhs*GKLn4!ODV+kAjA4%C!5^E6Dw+xa3U!^2go*(*Q zPA;ZjNu9&4Jfq>OkZ_yat+cP}warB2yN7S0rl}tgGN8KqNv3g^1@#)E2@1$c63Mee za@@$9WX8z~$eHG(Y{tur7N(@oYuc6Ow;G8<%&DxDH5R%Pu#jY|(y(P_nC?4)xA-of z&bJA+<4+haLJX)4bD_d1^l>?Rqo8-U$Q}P}f6cwILm?1=wti!CwagF1tQ2x*^une@ zSUHV^-db4gA|?iHsK|^<;jZmpaAf=EkY?PRibt`iJe?{+lvmB$+V~~nl%78MCED)b z4eQFUDFRdpGo=&O=AQAlGj|_YF?Uxyez$DtbrDSZXH{3Tti3GU3%v}K;>%ORw2iOO zRVqbm6H`v1UYmJg$H;$SNA|w5@i6J`p?#AbRoz%OjWWqStL=8GL_2SVIhVFC=pac- zfK%tH`LTJ0p><=Tg_4PU;J}B2z_>eOwdzYj z;uPsgf0*p&Imi!*^-QHFmPx3@yn|Mnr){#AXUTDrQUaa@ATWD(UrFavBr_wM=8Rj2 zcx;62SN&>(ko{^4=DL9Tr+E1m?9DE<`#*Vt5SI$fW5JLGzm;|SFd_J9OXS20mTVEo zN6q&y3`GfCX$sY`$2y};7dlG8W7RYEXF)!np<+y!NWZU`-%tiSlyf@&9PYw=Z(;(5 z7FF~I!5x=Rta$yWjJNQxij`S%JFEe%r)8vT!>3zxP_FRC(yPF^0)ssZ3EkQ$UTo%O zZCt(l#@2?;p`GX=H$zQy|KzQaRy=l;KZRDveMKV0JgVvH$gw%)60kNt%@){_ zl($vuB=)_Dj1s`nHA^c=+VY#Id!O!}VigVf*F{Wpi75S&2z+N_7ea94W+P&y!J&-$ ztN406#~EO0;JXFSh;-_(1W%+~P%#_1wYY&TtHEAg3r(q-^l@3aHtYz&)wp_|=R0wS z&4!BQHW9gNUVVa}=0;96ZP9l$Z6sYDx2W!+5bE8Jo<`t@*>a$vd{ZfNkYTV$G=0&G zU|+Rqi=#4Y18 z%|g@LWhAh?jF&T9tY#p6I_u8;AOGe`OWCOewuzN!-b!w>g;I>|YW`T#FKO!L9@6Os z+)NcM@CMg!ICOk?Hx$zWk)kUDcd<*(Z2P-7(6KGi|+<$5<(YcX=szVMI{PK zu<79J@2P()>KJX}PzSSsFB!Dcc)DQ-Ss53@PNHW0O=Qs#ml-D0LC`@3l4J2Cuoy!; zN35g1)7~|#E8Di1HdH+4kJd_383Rt6QP&N;>pdvaAq7t$)A{GAS)7p$j15okajTuc zT`JolRcpGjP^6ORkREJV?&0a6ne&T1h!%Cr3zPd-*LUHNg@h=rDBS{;Nft=M2$6*n z9wD(<17lgh<*NmV>RW<<{qw|ma5Ml3#5ah)gQDQ-e-;2x{iea2aQ$PVbrDZYbR^I# zkSIg{^1J)v;r>R)6_b$r8_^1J+);ew6Bu#Qj^|L^NB)p(B!Z=Y>~=$g^TM%|2>v4S zkOubxQ#(5y$B~6VK_90OA+f#?fkuCfO^n|#BBx|bRu0q^t%{ET{y4Cfnnja9h}Rmh zPaIY);o#CV)7`{0qLW~tc*B^7(4_aoAb{Tn1D98dE(j0SiDE?N7MV+c?C+|N((8}U zx?_*5nSjK^fME1cJL2Hev6C5E+{}@-MTkknP!M~B4fs(Bha;5%8EAA__T(ZD$|0o zG;bXAZrSU4A%=|x1lj>Q!R=g}If;OqlGf>q?<~q-6)+V*P8*~A0A$tI1CoJLpLiR` zA4s#S;mesp!SpM|5MHs=OGQ)rGSNnVIqSke-uyix5qF16w;)R?(0;eX*?hlZ6EZq1 z+m;X|kNQ=~#!ZQmjO9W&{`#VnDj*_Nh%$eX`USRxn6`o8pEOwp^02*u0yVYNwVq#S zjMB4&>vjMzFS=xLjV}`E(x7nXb6fo71>HZVsGNgkT*luDl_n%WuLc2

orKfDvLfXE5eZu zhcLw*X20}}2fE$7NYrIa$k4#1ttj4DxI4ZvP^k#GfmpQRi#PMko!K*MwTP6z3b|P@ z+I9Gdxyw1K3t6hzp}S~jpS0N;0aq3~ck{i4K)M@JFJ?A3-R5zyf?fT58 zlZFOttT~s$Kj^nziCY~3*H`yh$YhCNdMxCeMb5r1%Sv_ncL#Kick^>Sk9&D6GUBTA zS(%C})B_de`@XBBHLp_^)@oE>*B}}j`3v{9(;&tme_Hhr6|cwDty&wc)gmBU-!ZBW zJpBhH)F7OPP!9wNNuNDK%inW2fWK02!MuR>qkV|e>g3|C-p|J_KF`{IdXuXH)_s2z zH-{{{{93}u&%yKGanK16v7w3;2mE8Sk6{4GT!A;4slmg^twh{mV>v^dqoh0sg$h9$ z&iFCJ<%@hXO&v8}tM>i=4PU<&3&nN)LW_SCG(XWfl5uR=6;W((>6U2EKcHgT^;y}5 z917pUC1h>kc*hah7ch6H`QF8L{D~=1{ZKZtG@6Qcc{?nnp7@Ct@A6yXq_ajnRpuxB z&b7m<^jdYID$}}steaoOb5LcCb4iQSOgF$mY!^X5Q_zwkeIRJ43pr$ zql+#L5DiHHlxOClJ)GBbk}pFDhv+DX^En@kBxJWq>}O@peBuEgzjMzbEuad|8p4b7 z%Yr2sB50v#7XY^+mx*GB0UXn=vnAJyl!BzO5c6R6bwA#2$7?I-Vt|XIY>rFJ~uSRfBA~ZTg6}lxf^vb3oq|JzwoT8x) zf?-?e+UG_D2lmTWA{mw2XNZpM@k{4u0#X> zY=`Xvb6nN%A?EpgxU&HWZ9ER;77h*@x_WPyebq3(9BPU6uiEe`A~fgRpF304$Vbd& z2jSY62maEr*?_bQH3xP;b&6=52kx`9HmZg339~peMZ598j^f=h0!YK5&t&Uq3dgGi zKZOy(89@`K-K|b@)C4f1`+Wifk1oD#feOsPP`wI-z+~|Y=j4Bh_{u3p1Jo}~x1aJ^ zZ%14>AI=}0vZl> zh5cy)S>eS?rPz!{xs5bkGpH5ZyX+cU7Fl7jSq{STbZ1_-%H9PJ+)3~5z?`ujaa|E$ zRIzMn#X0+*5$h0vXkVHZi-w=NDyq&l?XtAHGwM~Kd!0DS^%py8=DLW}~ zP?Y8`{gRG{g`Yy+`aMO5#lh~seM=9q-0Ak6F;kd>L3695n7<-tDpf>&9NhD(bV#*`#oX<1$`YbUlD9~zrGB<%t~V}Nng z(@>NBG4)X~4AOgE;Bb2>Hr}BdSu1-mW=9`smmUSE6kMQ{$;)Y`^s~wpdNFZVOi!LN ztE+M-W(0n5>m`r@R243HoOyszc>QMr=N^5lR!ys;B9v14ynLM=l)?QXm}z?p$wD(v zpY&B7GanbL_c=u|^>!-Goc7~fsNn{5Pi!e;x%L(3x?t^)P6vnbiAG{kG$WFy$YhZS zIQmv<=Ten8scrhoPA4pokU@#XSFPKm&-$m7Lo_l&57QdThuyD<`NxMKYu>A~Raq`F`SE!1nKls#6;VvM zJsnY|c!4BVz|l{|y98ieOEv)lwM`&ap#?~{wCE3}{{Xe8>!qod?6roKSzWy~rzNku z)mRw`kS>y`;p(`$;juphM8w zl6q<3*pF?d?r#8buIb6h;UPEk?pLFy)18T?%ydk+CA%^E1QW1N->*2^OMK5sRmXYf z&brK7fWN>qMkvU|(d(_7Fvowy^pPL&TFf;LL!mbvd8{aD4BPS$iyVh1v+dN8#_@*Z zc9M@AF}6VE@R^ ztKg_;XX5qI=M!l6s2tzhBSs3nAYWsAVI>{6#hFb1d8P50%b}F@dEyN!d~12D;k62I zG~f)MQDL_e;-;RlUgENxSv$Fmo2t8(V7K`yd(&ihn9Yi|Sk&W6dvx*K4#+s&P3E#< z+B&y#*_6-HW_PU3bcE6e&Em&NUrgK#uqCeR_`8z5w++{a-ksc+^u^28kG#zi#S zi)HlfkJ$W%>0xN)3H1Lu7$V9S#F_0A`V?|xwkU#oJ2m$8_>URaNongoDF0i{byv%} zz#P|D#Q{#i8#5nch594bt8n(`l?^2UTU>(=KxcN>4etVVd7HOSOCK)x!_perSaY_J zpxMJHT;bp?H!nUu+}Oy4D|o6L$mXSmD;I$)s>qK#M*lzeUn8J2$Jl6AHq|z5h$joh zu*Ff`#Den-gr04iKf-^fsZ-G7{x%+ZaOu8X(*Yl`1OC~QyPni#K0zN}6W1Zf<$8#~ zahn#xQ$#tiBSXl{>yq*^ImHC_n`_SS6QcMsBfq>DXu=Htq0=F!g`fpH2UaHE6nEU< zqQ2*#azh#C9Op7OZQrWr`p#tB2%dJ*-S8QU<+5ky>g51&$@5o1q>kIor)5W+*M^!y ztiTfKf7~{&d)!CzF$%S%TSlcw>v-5WX*iaFEa`zAz&cv&KciLED^q6pM3j&^dI(h8Nnvy1(^(hZdgs>c-e z;Gt+lFWeR5d*Hh@afC$`GD@X#cTsll5>N$nGPgFj*Q-P0DTlCSCC&TCbsMx+bFD;= z-OBIv%&p;Uy^r|og8%p-gqzv$tD|e4YqzilB%zx{G@d-LhcfF6duYfTikx;l(a2H& zdpW#j;kE*85F4h@e3N{%cPk8hx^N28=!9nMxLjSM4k(j+;o~apZqaKtli25nG!3RTuy} z(pDCO&%x;_8y&YtEc|_^tGT4R2!tm5npp4+8W&dghE-@Q`^nclc0Fn5}-KsI}xz<>?*KxE-LF)3+ORCH@$d*@> z!nRzucs!Pih7az&d{ntqErca(=fibYc`&Dt*HXhCEbdHfuSFT)Y&45Y1X{>R&ba?L zUL5h(MWRrg)Pr>99sDg*o6f`N%1YU-oys=Y)zW3%`&Tn1E25KWS6qI^H$hK*Edrym z4J?{q%^&c0j-ohs+DX28vv6pmA z2BIqufsLBJAnhDpXb;X-O`9?ay_hoN#nj;#3z=Kb;3vN0qvgJOZLhd$i-b6vZCd#8 z%KnnYC}7;sKNm!fNalZyWXT9@as;KAP>bC=zQ4XlzZad`Dbk9*0_&GPxEPRWTs$+| zik9Bia6h;2HbXs^p0^A>YGewIIx&)a>1ZES;s5!QEio~O_WkH?(%agtsF^BRJ6?3I z()$8G>f9&lrpnw6j0xQ!%k%?VXia;(OngH$-PB>j)L=>>5~;sl0d*SPTMW<{mKjbu zBwYc)HGW}&;LRP((kPw5k<$!PAtPZ}*w|TFI62a8QXvWcUxAtiIS9c)@;_g}z%+ZM zEf$oqz0+T>$BmIALc(%ZXpDBp1&}(jxJw3Rl)=*T`Bc(dZF{^jE5ijFM$#l(r=Xp= zZ@nuuSkXwZK@4(v6r^M)nb1~o>B4rZ{>avBX-S5!5Xu-EiF68t?ZCslATX#*r-A|; zxEOYmvBH$1B&>B*9&?0D=&%9^psA4#Sif}efB0UkfLJ<&Bb}f*8C*=5rG&juYu^uH zKbcP_n($aFJ_C>N>gLD7$n99DcwUn^s`ye_o&>L(B$jvxkJ2_m9ft2{xrB$|C-xqvvxW18DQmPYT28#W6Bn)X{83^yadvZk*i_dI*3U5N*wW)QLmM zF{&$DOUIjt0eeHCvzu}>HB2eb0POYJ$<5DO1(FEs-oNv#yy zhZ|Q0QS+S}a?bhu2IB#1b@2Dtmas4$VytcV`E7XKhW{=#ahVW_{hdrFs?zJ6Mql=g zlSdCXPV2eP^5Hy8Tf%HQklSeM-GjAL$hs+msC8C%zza0-@W9*tq@}DBWxoqaZCWIGZF9qR*%d6TSU_2!z|;>xXf6m zyWq(oFlIoF^3Y->x>E5ZT;c%L{}cLidoEg?IvBwUq02>oR4g^;2X>dRFI{yaR+I&4pvum}&B-ec5R>HO;t`XO z4Sc1~usr|JL=KB^mh~eR3E!b@-QSaV#_TmMToKTG^*6 z1rP}_d_GE4K9;PS&YqniZ8CsYO@vYnz_~0!zg!WTcdaGJAwtdpU~hIJeLu~aFGV|V zM8BQ~@2UUkUyo?K9u$BR2v`px#R+4?2~xocaZV4EiRe!TP9Ajkc@3Diu4R;H zuZ)+K@UHx!lLJ+r5~pIRIf|xYt^EAZ(ouDEn_F!3c`=rwzwqDF(NPD!Pc3HtE7MF} zUFq{{XfF72?@iXd3R0Nm6+(`qx@_;SK@uqPF`1)1%@~i9v coXkpq>OYT8gU^K|fM@4mgQujFP?CiIKVUI`Z2$lO diff --git a/doc/src/DimRed/DimRed.do.txt b/doc/src/DimRed/DimRed.do.txt index cdeab6903..9e864338d 100644 --- a/doc/src/DimRed/DimRed.do.txt +++ b/doc/src/DimRed/DimRed.do.txt @@ -332,40 +332,42 @@ applications. ===== Basic ideas of the Principal Component Analysis (PCA) ===== We have a data set defined by a design/feature matrix $\bm{X}$ (see below for its definition) -* So each data point is determined by $p$ extrinsic (measurement) variables +* Each data point is determined by $p$ extrinsic (measurement) variables * We may want to ask the following question: Are there fewer intrinsic variables (say $d << p$) that still approximately describe the data? -* If so, these intrinsic variables may tell us something important and finding these intrinsic variables is what dimension reduction methods do +* If so, these intrinsic variables may tell us something important and finding these intrinsic variables is what dimension reduction methods do. !split ===== Introducing the Covariance and Correlation functions ===== +Before we discuss the PCA theorem, we need to remind ourselves about the definition of the covariance and the correlation function. + Suppose we have defined two vectors $\hat{x}$ and $\hat{y}$ with $n$ elements each. The covariance matrix $\bm{C}$ is defined as !bt \[ -\bm{C}[\bm{x},\bm{y}] = \begin{bmatrix} cov[\bm{x},\bm{x}] & cov[\bm{x},\bm{y}] \\ - cov[\bm{y},\bm{x}] & cov[\bm{y},\bm{y}] \\ +\bm{C}[\bm{x},\bm{y}] = \begin{bmatrix} \mathrm{cov}[\bm{x},\bm{x}] & \mathrm{cov}[\bm{x},\bm{y}] \\ + \mathrm{cov}[\bm{y},\bm{x}] & \mathrm{cov}[\bm{y},\bm{y}] \\ \end{bmatrix}, \] !et where for example !bt \[ -cov[\bm{x},\bm{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). +\mathrm{cov}[\bm{x},\bm{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). \] !et -With this definition and recalling that the variance is +With this definition and recalling that the variance is defined as !bt \[ -var[\bm{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2, +\mathrm{var}[\bm{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2, \] !et -we can rewrite the covariance matrix in this case as +we can rewrite the covariance matrix as !bt \[ -\bm{C}[\bm{x},\bm{y}] = \begin{bmatrix} var[\bm{x}] & cov[\bm{x},\bm{y}] \\ - cov[\bm{x},\bm{y}] & var[\bm{y}] \\ +\bm{C}[\bm{x},\bm{y}] = \begin{bmatrix} \mathrm{var}[\bm{x}] & \mathrm{cov}[\bm{x},\bm{y}] \\ + \mathrm{cov}[\bm{x},\bm{y}] & \mathrm{var}[\bm{y}] \\ \end{bmatrix}, \] !et @@ -378,19 +380,19 @@ correlation function !bt \[ -corr[\bm{x},\bm{y}]=\frac{cov[\bm{x},\bm{y}]}{\sqrt{var[\bm{x}]\var[\bm{y}]}}. +\mathrm{corr}[\bm{x},\bm{y}]=\frac{\mathrm{cov}[\bm{x},\bm{y}]}{\sqrt{\mathrm{var}[\bm{x}] \mathrm{var}[\bm{y}]}}. \] !et -The correlation function is then given by values $corr[\bm{x},\bm{y}] +The correlation function is then given by values $\mathrm{corr}[\bm{x},\bm{y}] \in [-1,1]$. This avoids eventual problems with too large values. We can then define the correlation matrix for the two vectors $\bm{x}$ and $\bm{y}$ as !bt \[ -\bm{K}[\bm{x},\bm{y}] = \begin{bmatrix} 1 & corr[\bm{x},\bm{y}] \\ - corr[\bm{y},\bm{x}] & 1 \\ +\bm{K}[\bm{x},\bm{y}] = \begin{bmatrix} 1 & \mathrm{corr}[\bm{x},\bm{y}] \\ + \mathrm{corr}[\bm{y},\bm{x}] & 1 \\ \end{bmatrix}, \] !et @@ -432,12 +434,12 @@ With these definitions, we can now rewrite our $2\times 2$ correaltion/covarianc !bt \[ \bm{C}[\bm{x}] = \begin{bmatrix} -var[\bm{x}_0] & cov[\bm{x}_0,\bm{x}_1] & cov[\bm{x}_0,\bm{x}_2] & \dots & \dots & cov[\bm{x}_0,\bm{x}_{p-1}]\\ -cov[\bm{x}_1,\bm{x}_0] & var[\bm{x}_1] & cov[\bm{x}_1,\bm{x}_2] & \dots & \dots & cov[\bm{x}_1,\bm{x}_{p-1}]\\ -cov[\bm{x}_2,\bm{x}_0] & cov[\bm{x}_2,\bm{x}_1] & var[\bm{x}_2] & \dots & \dots & cov[\bm{x}_2,\bm{x}_{p-1}]\\ +\mathrm{var}[\bm{x}_0] & \mathrm{cov}[\bm{x}_0,\bm{x}_1] & \mathrm{cov}[\bm{x}_0,\bm{x}_2] & \dots & \dots & \mathrm{cov}[\bm{x}_0,\bm{x}_{p-1}]\\ +\mathrm{cov}[\bm{x}_1,\bm{x}_0] & \mathrm{var}[\bm{x}_1] & \mathrm{cov}[\bm{x}_1,\bm{x}_2] & \dots & \dots & \mathrm{cov}[\bm{x}_1,\bm{x}_{p-1}]\\ +\mathrm{cov}[\bm{x}_2,\bm{x}_0] & \mathrm{cov}[\bm{x}_2,\bm{x}_1] & \mathrm{var}[\bm{x}_2] & \dots & \dots & \mathrm{cov}[\bm{x}_2,\bm{x}_{p-1}]\\ \dots & \dots & \dots & \dots & \dots & \dots \\ \dots & \dots & \dots & \dots & \dots & \dots \\ -cov[\bm{x}_{p-1},\bm{x}_0] & cov[\bm{x}_{p-1},\bm{x}_1] & cov[\bm{x}_{p-1},\bm{x}_{2}] & \dots & \dots & var[\bm{x}_{p-1}]\\ +\mathrm{cov}[\bm{x}_{p-1},\bm{x}_0] & \mathrm{cov}[\bm{x}_{p-1},\bm{x}_1] & \mathrm{cov}[\bm{x}_{p-1},\bm{x}_{2}] & \dots & \dots & \mathrm{var}[\bm{x}_{p-1}]\\ \end{bmatrix}, \] !et @@ -445,12 +447,12 @@ and the correlation matrix !bt \[ \bm{K}[\bm{x}] = \begin{bmatrix} -1 & corr[\bm{x}_0,\bm{x}_1] & corr[\bm{x}_0,\bm{x}_2] & \dots & \dots & corr[\bm{x}_0,\bm{x}_{p-1}]\\ -corr[\bm{x}_1,\bm{x}_0] & 1 & corr[\bm{x}_1,\bm{x}_2] & \dots & \dots & corr[\bm{x}_1,\bm{x}_{p-1}]\\ -corr[\bm{x}_2,\bm{x}_0] & corr[\bm{x}_2,\bm{x}_1] & 1 & \dots & \dots & corr[\bm{x}_2,\bm{x}_{p-1}]\\ +1 & \mathrm{corr}[\bm{x}_0,\bm{x}_1] & \mathrm{corr}[\bm{x}_0,\bm{x}_2] & \dots & \dots & \mathrm{corr}[\bm{x}_0,\bm{x}_{p-1}]\\ +\mathrm{corr}[\bm{x}_1,\bm{x}_0] & 1 & \mathrm{corr}[\bm{x}_1,\bm{x}_2] & \dots & \dots & \mathrm{corr}[\bm{x}_1,\bm{x}_{p-1}]\\ +\mathrm{corr}[\bm{x}_2,\bm{x}_0] & \mathrm{corr}[\bm{x}_2,\bm{x}_1] & 1 & \dots & \dots & \mathrm{corr}[\bm{x}_2,\bm{x}_{p-1}]\\ \dots & \dots & \dots & \dots & \dots & \dots \\ \dots & \dots & \dots & \dots & \dots & \dots \\ -corr[\bm{x}_{p-1},\bm{x}_0] & corr[\bm{x}_{p-1},\bm{x}_1] & corr[\bm{x}_{p-1},\bm{x}_{2}] & \dots & \dots & 1\\ +\mathrm{corr}[\bm{x}_{p-1},\bm{x}_0] & \mathrm{corr}[\bm{x}_{p-1},\bm{x}_1] & \mathrm{corr}[\bm{x}_{p-1},\bm{x}_{2}] & \dots & \dots & 1\\ \end{bmatrix}, \] !et