Added dimensionality red methods

doc/Programs/JupyterFiles/Examples/My Own Examples/.ipynb_checkpoints/Functions in mglearn-checkpoint.ipynb

@@ -0,0 +1,107 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import mglearn\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import os\n",
+    "from scipy import signal\n",
+    "from sklearn.datasets import load_boston\n",
+    "from sklearn.preprocessing import MinMaxScaler, PolynomialFeatures\n",
+    "from mglearn.make_blobs import make_blobs\n",
+    "\n",
+    "#DATA_PATH = os.path.join(os.path.dirname(__file__), \"data\")\n",
+    "\n",
+    "\n",
+    "def make_forge():\n",
+    "    # a carefully hand-designed dataset lol\n",
+    "    X, y = make_blobs(centers=2, random_state=4, n_samples=30)\n",
+    "    y[np.array([7, 27])] = 0\n",
+    "    mask = np.ones(len(X), dtype=np.bool)\n",
+    "    mask[np.array([0, 1, 5, 26])] = 0\n",
+    "    X, y = X[mask], y[mask]\n",
+    "    return X, y\n",
+    "\n",
+    "\n",
+    "def make_wave(n_samples=100):\n",
+    "    rnd = np.random.RandomState(42)\n",
+    "    x = rnd.uniform(-3, 3, size=n_samples)\n",
+    "    y_no_noise = (np.sin(4 * x) + x)\n",
+    "    y = (y_no_noise + rnd.normal(size=len(x))) / 2\n",
+    "    return x.reshape(-1, 1), y\n",
+    "\n",
+    "\n",
+    "def load_extended_boston():\n",
+    "    boston = load_boston()\n",
+    "    X = boston.data\n",
+    "\n",
+    "    X = MinMaxScaler().fit_transform(boston.data)\n",
+    "    X = PolynomialFeatures(degree=2, include_bias=False).fit_transform(X)\n",
+    "    return X, boston.target\n",
+    "\n",
+    "\n",
+    "def load_citibike():\n",
+    "    data_mine = pd.read_csv(os.path.join(DATA_PATH, \"citibike.csv\"))\n",
+    "    data_mine['one'] = 1\n",
+    "    data_mine['starttime'] = pd.to_datetime(data_mine.starttime)\n",
+    "    data_starttime = data_mine.set_index(\"starttime\")\n",
+    "    data_resampled = data_starttime.resample(\"3h\").sum().fillna(0)\n",
+    "    return data_resampled.one\n",
+    "\n",
+    "\n",
+    "def make_signals():\n",
+    "    # fix a random state seed\n",
+    "    rng = np.random.RandomState(42)\n",
+    "    n_samples = 2000\n",
+    "    time = np.linspace(0, 8, n_samples)\n",
+    "    # create three signals\n",
+    "    s1 = np.sin(2 * time)  # Signal 1 : sinusoidal signal\n",
+    "    s2 = np.sign(np.sin(3 * time))  # Signal 2 : square signal\n",
+    "    s3 = signal.sawtooth(2 * np.pi * time)  # Signal 3: saw tooth signal\n",
+    "\n",
+    "    # concatenate the signals, add noise\n",
+    "    S = np.c_[s1, s2, s3]\n",
+    "    S += 0.2 * rng.normal(size=S.shape)\n",
+    "\n",
+    "    S /= S.std(axis=0)  # Standardize data\n",
+    "    S -= S.min()\n",
+    "    return S\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

doc/src/DimRed/DimRed.do.txt

@@ -0,0 +1,206 @@
+TITLE: Data Analysis and Machine Learning: Dimensionality Reduction
+AUTHOR: Morten Hjorth-Jensen {copyright, 1999-present|CC BY-NC} at Department of Physics, University of Oslo & Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
+DATE: today
+
+
+!split
+===== Reducing the number of degrees of freedom, overarching view  =====
+!bblock
+
+Many Machine Learning problems involve thousands or even millions of features for each training
+instance. Not only does this make training extremely slow, it can also make it much harder to find a good
+solution, as we will see. This problem is often referred to as the curse of dimensionality.
+Fortunately, in real-world problems, it is often possible to reduce the number of features considerably,
+turning an intractable problem into a tractable one.
+
+and we will go through three of the most popular dimensionality
+reduction techniques: PCA, Kernel PCA, and LLE.
+
+!eblock
+
+
+
+!split
+===== Principal Component Analysis =====
+!bblock
+Principal Component Analysis (PCA) is by far the most popular dimensionality reduction algorithm.
+First it identifies the hyperplane that lies closest to the data, and then it projects the data onto it.
+
+The following Python code uses NumPy’s svd() function to obtain all the principal components of the
+training set, then extracts the first two PCs:
+X_centered = X - X.mean(axis=0)
+U, s, V = np.linalg.svd(X_centered)
+c1 = V.T[:, 0]
+c2 = V.T[:, 1]
+
+
+PCA assumes that the dataset is centered around the origin. As we will see, Scikit-Learn’s PCA classes take care of centering
+the data for you. However, if you implement PCA yourself (as in the preceding example), or if you use other libraries, don’t
+forget to center the data first.
+
+Once you have identified all the principal components, you can reduce the dimensionality of the dataset
+down to d dimensions by projecting it onto the hyperplane defined by the first d principal components.
+Selecting this hyperplane ensures that the projection will preserve as much variance as possible. For
+example, in Figure 8-2 the 3D dataset is projected down to the 2D plane defined by the first two principal
+components, preserving a large part of the dataset’s variance. As a result, the 2D projection looks very
+much like the original 3D dataset.
+
+W2 = V.T[:, :2]
+X2D = X_centered.dot(W2)
+
+Scikit-Learn’s PCA class implements PCA using SVD decomposition just like we did before. The
+following code applies PCA to reduce the dimensionality of the dataset down to two dimensions (note
+that it automatically takes care of centering the data):
+from sklearn.decomposition import PCA
+pca = PCA(n_components = 2)
+X2D = pca.fit_transform(X)
+After fitting the PCA transformer to the dataset, you can access the principal components using the
+components_ variable (note that it contains the PCs as horizontal vectors, so, for example, the first
+principal component is equal to pca.components_.T[:, 0]).
+
+Another very useful piece of information is the explained variance ratio of each principal component,
+available via the explained_variance_ratio_ variable. It indicates the proportion of the dataset’s
+variance that lies along the axis of each principal component. For example, let’s look at the explained
+variance ratios of the first two components of the 3D dataset represented in Figure 8-2:
+>>> print(pca.explained_variance_ratio_)
+array([ 0.84248607, 0.14631839])
+This tells you that 84.2% of the dataset’s variance lies along the first axis, and 14.6% lies along the
+second axis. This leaves less than 1.2% for the third axis, so it is reasonable to assume that it probably
+carries little information.
+
+
+Instead of arbitrarily choosing the number of dimensions to reduce down to, it is generally preferable to
+choose the number of dimensions that add up to a sufficiently large portion of the variance (e.g., 95%).
+Unless, of course, you are reducing dimensionality for data visualization — in that case you will
+generally want to reduce the dimensionality down to 2 or 3.
+The following code computes PCA without reducing dimensionality, then computes the minimum number
+of dimensions required to preserve 95% of the training set’s variance:
+pca = PCA()
+pca.fit(X)
+cumsum = np.cumsum(pca.explained_variance_ratio_)
+d = np.argmax(cumsum >= 0.95) + 1
+You could then set n_components=d and run PCA again. However, there is a much better option: instead
+of specifying the number of principal components you want to preserve, you can set n_components to be
+a float between 0.0 and 1.0, indicating the ratio of variance you wish to preserve:
+pca = PCA(n_components=0.95)
+X_reduced = pca.fit_transform(X)
+
+
+
+Obviously after dimensionality reduction, the training set takes up much less space. For example, try
+applying PCA to the MNIST dataset while preserving 95% of its variance. You should find that each
+instance will have just over 150 features, instead of the original 784 features. So while most of the
+variance is preserved, the dataset is now less than 20% of its original size! This is a reasonable
+compression ratio, and you can see how this can speed up a classification algorithm (such as an SVM
+classifier) tremendously.
+It is also possible to decompress the reduced dataset back to 784 dimensions by applying the inverse
+transformation of the PCA projection. Of course this won’t give you back the original data, since the
+projection lost a bit of information (within the 5% variance that was dropped), but it will likely be quite
+close to the original data. The mean squared distance between the original data and the reconstructed data
+(compressed and then decompressed) is called the reconstruction error. For example, the following code
+compresses the MNIST dataset down to 154 dimensions, then uses the inverse_transform() method to
+decompress it back to 784 dimensions. Figure 8-9 shows a few digits from the original training set (on the
+left), and the corresponding digits after compression and decompression. You can see that there is a slight
+image quality loss, but the digits are still mostly intact.
+pca = PCA(n_components = 154)
+X_mnist_reduced = pca.fit_transform(X_mnist)
+X_mnist_recovered = pca.inverse_transform(X_mnist_reduced)
+Figure
+
+
+
+Incremental PCA
+One problem with the preceding implementation of PCA is that it requires the whole training set to fit in
+memory in order for the SVD algorithm to run. Fortunately, Incremental PCA (IPCA) algorithms have
+been developed: you can split the training set into mini-batches and feed an IPCA algorithm one minibatch
+at a time. This is useful for large training sets, and also to apply PCA online (i.e., on the fly, as new
+instances arrive).
+The following code splits the MNIST dataset into 100 mini-batches (using NumPy’s array_split()
+function) and feeds them to Scikit-Learn’s IncrementalPCA class5 to reduce the dimensionality of the
+MNIST dataset down to 154 dimensions (just like before). Note that you must call the partial_fit()
+method with each mini-batch rather than the fit() method with the whole training set:
+from sklearn.decomposition import IncrementalPCA
+n_batches = 100
+inc_pca = IncrementalPCA(n_components=154)
+for X_batch in np.array_split(X_mnist, n_batches):
+inc_pca.partial_fit(X_batch)
+X_mnist_reduced = inc_pca.transform(X_mnist)
+
+
+
+Alternatively, you can use NumPy’s memmap class, which allows you to manipulate a large array stored in
+a binary file on disk as if it were entirely in memory; the class loads only the data it needs in memory,
+when it needs it. Since the IncrementalPCA class uses only a small part of the array at any given time,
+the memory usage remains under control. This makes it possible to call the usual fit() method, as you
+can see in the following code:
+X_mm = np.memmap(filename, dtype="float32", mode="readonly", shape=(m, n))
+batch_size = m // n_batches
+inc_pca = IncrementalPCA(n_components=154, batch_size=batch_size)
+inc_pca.fit(X_mm)
+
+
+Randomized PCA
+Scikit-Learn offers yet another option to perform PCA, called Randomized PCA. This is a stochastic
+algorithm that quickly finds an approximation of the first d principal components. Its computational
+complexity is O(m × d2) + O(d3), instead of O(m × n2) + O(n3), so it is dramatically faster than the
+previous algorithms when d is much smaller than n.
+rnd_pca = PCA(n_components=154, svd_solver="randomized")
+X_reduced = rnd_pca.fit_transform(X_mnist)
+
+
+!eblock
+
+
+!split
+===== Kernel PCA =====
+!bblock
+
+Kernel PCA
+The kernel trick is a mathematical technique that implicitly maps instances into a
+very high-dimensional space (called the feature space), enabling nonlinear classification and regression
+with Support Vector Machines. Recall that a linear decision boundary in the high-dimensional feature
+space corresponds to a complex nonlinear decision boundary in the original space.
+It turns out that the same trick can be applied to PCA, making it possible to perform complex nonlinear
+projections for dimensionality reduction. This is called Kernel PCA (kPCA). It is often good at
+preserving clusters of instances after projection, or sometimes even unrolling datasets that lie close to a
+twisted manifold.
+For example, the following code uses Scikit-Learn’s KernelPCA class to perform kPCA with an
+from sklearn.decomposition import KernelPCA
+rbf_pca = KernelPCA(n_components = 2, kernel="rbf", gamma=0.04)
+X_reduced = rbf_pca.fit_transform(X)
+Figure 8-
+
+!eblock 
+
+
+!split
+===== LLE =====
+
+Locally Linear Embedding (LLE)8 is another very powerful nonlinear dimensionality reduction
+(NLDR) technique. It is a Manifold Learning technique that does not rely on projections like the previous
+algorithms. In a nutshell, LLE works by first measuring how each training instance linearly relates to its
+closest neighbors (c.n.), and then looking for a low-dimensional representation of the training set where
+these local relationships are best preserved (more details shortly). This makes it particularly good at
+unrolling twisted manifolds, especially when there is not too much noise.
+
+
+
+!split
+=====  Other techniques =====
+
+
+There are many other dimensionality reduction techniques, several of which are available in Scikit-Learn.
+Here are some of the most popular:
+Multidimensional Scaling (MDS) reduces dimensionality while trying to preserve the distances
+between the instances (see Figure 8-13).
+Isomap creates a graph by connecting each instance to its nearest neighbors, then reduces
+dimensionality while trying to preserve the geodesic distances9 between the instances.
+t-Distributed Stochastic Neighbor Embedding (t-SNE) reduces dimensionality while trying to keep
+similar instances close and dissimilar instances apart. It is mostly used for visualization, in
+particular to visualize clusters of instances in high-dimensional space (e.g., to visualize the MNIST
+images in 2D).
+Linear Discriminant Analysis (LDA) is actually a classification algorithm, but during training it
+learns the most discriminative axes between the classes, and these axes can then be used to define a
+hyperplane onto which to project the data. The benefit is that the projection will keep classes as far
+apart as possible, so LDA is a good technique to reduce dimensionality before running another
+classification algorithm such as an SVM classifier

doc/src/DimRed/clean.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+doconce clean
+rm -rf *.pdf *.tex ipynb*.tar.gz *.html ._*.html *~ reveal.js Trash README.txt

doc/src/DimRed/make.sh

@@ -0,0 +1,95 @@
+#!/bin/sh
+set -x
+
+function system {
+  "$@"
+  if [ $? -ne 0 ]; then
+    echo "make.sh: unsuccessful command $@"
+    echo "abort!"
+    exit 1
+  fi
+}
+
+if [ $# -eq 0 ]; then
+echo 'bash make.sh slides1|slides2'
+exit 1
+fi
+
+name=$1
+rm -f *.tar.gz
+
+opt="--encoding=utf-8"
+# Note: Makefile examples contain constructions like ${PROG} which
+# looks like Mako constructions, but they are not. Use --no_mako
+# to turn off Mako processing.
+opt="--no_mako"
+
+rm -f *.aux
+
+
+html=${name}-reveal
+system doconce format html $name --pygments_html_style=perldoc --keep_pygments_html_bg --html_links_in_new_window --html_output=$html $opt
+system doconce slides_html $html reveal --html_slide_theme=beige
+
+# Plain HTML documents
+
+html=${name}-solarized
+system doconce format html $name --pygments_html_style=perldoc --html_style=solarized3 --html_links_in_new_window --html_output=$html $opt
+system doconce split_html $html.html --method=space10
+
+html=${name}
+system doconce format html $name --pygments_html_style=default --html_style=bloodish --html_links_in_new_window --html_output=$html $opt
+system doconce split_html $html.html --method=space10
+
+# Bootstrap style
+html=${name}-bs
+system doconce format html $name --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=$html $opt
+system doconce split_html $html.html --method=split --pagination --nav_button=bottom
+
+# IPython notebook
+system doconce format ipynb $name $opt
+
+
+# Ordinary plain LaTeX document
+rm -f *.aux  # important after beamer
+system doconce format pdflatex $name --minted_latex_style=trac --latex_admon=paragraph $opt
+system doconce ptex2tex $name envir=minted
+# Add special packages
+doconce subst "% Add user's preamble" "\g<1>\n\\usepackage{simplewick}" $name.tex
+doconce replace 'section{' 'section*{' $name.tex
+pdflatex -shell-escape $name
+pdflatex -shell-escape $name
+mv -f $name.pdf ${name}-minted.pdf
+cp $name.tex ${name}-plain-minted.tex
+
+
+
+# Publish
+dest=../../pub
+if [ ! -d $dest/$name ]; then
+mkdir $dest/$name
+mkdir $dest/$name/pdf
+mkdir $dest/$name/html
+mkdir $dest/$name/ipynb
+fi
+cp ${name}*.pdf $dest/$name/pdf
+cp -r ${name}*.html ._${name}*.html reveal.js $dest/$name/html
+
+# Figures: cannot just copy link, need to physically copy the files
+if [ -d fig-${name} ]; then
+if [ ! -d $dest/$name/html/fig-$name ]; then
+mkdir $dest/$name/html/fig-$name
+fi
+cp -r fig-${name}/* $dest/$name/html/fig-$name
+fi
+
+cp ${name}.ipynb $dest/$name/ipynb
+ipynb_tarfile=ipynb-${name}-src.tar.gz
+if [ ! -f ${ipynb_tarfile} ]; then
+cat > README.txt <<EOF
+This IPython notebook ${name}.ipynb does not require any additional
+programs.
+EOF
+tar czf ${ipynb_tarfile} README.txt
+fi
+cp ${ipynb_tarfile} $dest/$name/ipynb

doc/web/course.do.txt

@@ -10,17 +10,18 @@ AUTHOR: "Morten Hjorth-Jensen":"http://mhjgit.github.io/info/doc/web/" at Depart
 
 <%
 pub_url = 'https://compphysics.github.io/MachineLearning/doc/pub'
-published = ['Intro2Course', 'Introduction', 'How2ReadData', 'Linalg', 'Statistics', 'Splines', 'Regression', 'LogReg', 'NeuralNet', 'Bayesian', 'DecisionTrees', 'svm', 'BM',]
+published = ['Intro2Course', 'Introduction', 'How2ReadData', 'Linalg', 'Statistics', 'Splines', 'Regression', 'LogReg', 'NeuralNet', 'DimRed', 'Bayesian', 'DecisionTrees', 'svm', 'BM',]
 chapters = {
  'Intro2Course': 'Basic introduction to the course with schedule etc',
  'Introduction': 'Introduction to Data Analysis and Machine Learning',
  'How2ReadData': 'Getting started with Machine Learning',
  'Linalg': 'Review of central linear algebra elements',
  'Statistics': 'Monte Carlo methods and elements of probability theory',
- 'Splines': 'Splines and Gradient methods',
+ 'Splines': 'Gradient methods',
  'Regression': 'Regression Methods',
  'LogReg': 'Logistic Regression',
  'NeuralNet': 'Neural Networks',
+ 'DimRed': 'Reduction of dimensionality',
  'Bayesian': 'Elements of Bayesian theory',
  'DecisionTrees': 'Decision trees, from simple to random ones',
  'svm': 'Support Vector Machines',